From 71b71b8156008bd52a99fda210cb7f310695e470 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 02:08:31 -0700 Subject: [PATCH 01/29] feat: trace post processing and Valkey sessions (#345) * feat: trace post processing and valkey sessions * docs: align orchestrator runtime pin * docs: codify telemetry boundary * fix: normalize OTLP trace endpoint * chore: pin orchestrator telemetry fix * fix: keep provider details out of ingestion ledger * fix: propagate W3C trace context to gateway --- AGENTS.md | 12 + backend/app/activity_stream.py | 46 ++- backend/app/analysis_run_outbox.py | 26 +- backend/app/analysis_run_worker.py | 11 +- backend/app/main.py | 2 + backend/app/post_content_queue.py | 24 +- backend/app/post_content_worker.py | 21 +- docker-compose.yml | 4 + docker/contextual-orchestrator/Dockerfile | 6 +- .../0083-orchestrator-runtime-commit-pin.md | 2 +- docs/adr/0122-otel-session-observability.md | 54 ++++ docs/doctoring/OPENTELEMETRY_REFERENCES.md | 27 ++ lineageweave/http_client.py | 34 ++- lineageweave/observability.py | 135 +++++++++ pyproject.toml | 3 + tests/test_observability.py | 58 ++++ tests/test_post_content_worker.py | 8 +- uv.lock | 269 ++++++++++++++++++ 18 files changed, 695 insertions(+), 47 deletions(-) create mode 100644 docs/adr/0122-otel-session-observability.md create mode 100644 docs/doctoring/OPENTELEMETRY_REFERENCES.md create mode 100644 lineageweave/observability.py create mode 100644 tests/test_observability.py diff --git a/AGENTS.md b/AGENTS.md index 1728f9e61..0d1206f7d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -129,6 +129,18 @@ contextual-orchestrator owns model discovery and selection. region-level evidence; never show an internal LLM instruction such as `This post is an image` to a buyer. +## Observability boundary + +- Follow governance-risk-compliance ADR 0009 and LineageWeave ADR 0122 for + OpenTelemetry. Use `OTEL_SERVICE_NAME` and + `OTEL_EXPORTER_OTLP_ENDPOINT`; exporting is opt-in and provider-neutral. +- Correlate one post's HTTP, contextual-orchestrator, and Valkey work with the + existing post-scoped session metadata. Do not create an ad hoc session table. +- Telemetry may contain bounded operation, route-template, service-peer, and + correlation attributes, but never post body, prompt, answer, source content, + actor or tenant identifiers, credentials, raw stream keys, or provider + responses. GRC remains the control/evidence owner. + ## Source parsing and semantic units - Preserve the source representation and provenance, then derive semantic diff --git a/backend/app/activity_stream.py b/backend/app/activity_stream.py index a47bf9343..56311382b 100644 --- a/backend/app/activity_stream.py +++ b/backend/app/activity_stream.py @@ -14,6 +14,8 @@ import redis.asyncio as redis from fastapi import Request +from lineageweave.observability import traced + def create_valkey_client(url: str) -> redis.Redis: """One shared async client for the process, mirroring db.create_pool.""" @@ -66,12 +68,16 @@ async def publish_activity_event( ``approximate=True``) so one very active post's stream can't grow without bound -- the panel only ever shows the most recent 50 anyway. """ - return await client.xadd( - _stream_key(post_id), - _activity_fields(event_type, actor_account_id, summary), - maxlen=1000, - approximate=True, - ) + with traced( + "lineageweave.valkey.activity_xadd", + {"db.system": "redis", "db.operation.name": "xadd", "lineageweave.stream.kind": "activity"}, + ): + return await client.xadd( + _stream_key(post_id), + _activity_fields(event_type, actor_account_id, summary), + maxlen=1000, + approximate=True, + ) def publish_activity_event_sync( @@ -83,20 +89,32 @@ def publish_activity_event_sync( ) -> str | None: """Sync ``XADD`` for ``make seed``. Returns None if ``summary`` is already on the stream.""" key = _stream_key(post_id) - existing = client.xrevrange(key, count=50) + with traced( + "lineageweave.valkey.activity_xrevrange", + {"db.system": "redis", "db.operation.name": "xrevrange", "lineageweave.stream.kind": "activity"}, + ): + existing = client.xrevrange(key, count=50) if any(fields.get("summary") == summary for _entry_id, fields in existing): return None - return client.xadd( - key, - _activity_fields(event_type, str(actor_account_id), summary), - maxlen=1000, - approximate=True, - ) + with traced( + "lineageweave.valkey.activity_xadd", + {"db.system": "redis", "db.operation.name": "xadd", "lineageweave.stream.kind": "activity"}, + ): + return client.xadd( + key, + _activity_fields(event_type, str(actor_account_id), summary), + maxlen=1000, + approximate=True, + ) async def read_activity_events(client: redis.Redis, post_id: str, count: int = 50) -> list[dict[str, Any]]: """The post's most recent events, newest first.""" - entries = await client.xrevrange(_stream_key(post_id), count=count) + with traced( + "lineageweave.valkey.activity_xrevrange", + {"db.system": "redis", "db.operation.name": "xrevrange", "lineageweave.stream.kind": "activity"}, + ): + entries = await client.xrevrange(_stream_key(post_id), count=count) return [ { "event_id": entry_id, diff --git a/backend/app/analysis_run_outbox.py b/backend/app/analysis_run_outbox.py index 487948ba8..fff121cc7 100644 --- a/backend/app/analysis_run_outbox.py +++ b/backend/app/analysis_run_outbox.py @@ -13,6 +13,8 @@ import redis.asyncio as redis +from lineageweave.observability import traced + OUTBOX_STREAM_KEY = "analysis-run-outbox" _CLAIMED = "analysis_outbox_claimed" _DELIVERED = "analysis_outbox_delivered" @@ -69,16 +71,20 @@ async def publish_outbox_event( if client is None: return None try: - entry_id = await client.xadd( - OUTBOX_STREAM_KEY, - outbox_stream_fields( - analysis_run_id=analysis_run_id, - work_kind_code=work_kind_code, - request_sha256=request_sha256, - ), - maxlen=1000, - approximate=True, - ) + with traced( + "lineageweave.valkey.analysis_outbox_xadd", + {"db.system": "redis", "db.operation.name": "xadd", "lineageweave.stream.kind": "analysis_outbox", "lineageweave.work_kind": work_kind_code}, + ): + entry_id = await client.xadd( + OUTBOX_STREAM_KEY, + outbox_stream_fields( + analysis_run_id=analysis_run_id, + work_kind_code=work_kind_code, + request_sha256=request_sha256, + ), + maxlen=1000, + approximate=True, + ) except redis.RedisError: return None return str(entry_id) diff --git a/backend/app/analysis_run_worker.py b/backend/app/analysis_run_worker.py index 43b8d17b7..cb842b67d 100644 --- a/backend/app/analysis_run_worker.py +++ b/backend/app/analysis_run_worker.py @@ -12,6 +12,7 @@ from uuid import UUID from lineageweave.adjudication_client import AdjudicationClient +from lineageweave.observability import traced from lineageweave.tepp_client import TeppClient from backend.app.analysis_run_outbox import OUTBOX_STREAM_KEY @@ -31,7 +32,15 @@ async def consume_analysis_run_stream_once( Invalid or stale entries are acknowledged by advancing the cursor; the durable PostgreSQL outbox remains available for a later explicit retry. """ - batches = await client.xread({OUTBOX_STREAM_KEY: last_id}, count=10, block=1000) + with traced( + "lineageweave.valkey.analysis_outbox_xread", + { + "db.system": "redis", + "db.operation.name": "xread", + "lineageweave.stream.kind": "analysis_outbox", + }, + ): + batches = await client.xread({OUTBOX_STREAM_KEY: last_id}, count=10, block=1000) for _stream_name, entries in batches: for entry_id, fields in entries: analysis_run_id = str(fields.get("analysis_run_id", "")).strip() diff --git a/backend/app/main.py b/backend/app/main.py index fb943315f..46dea0e09 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -186,6 +186,7 @@ is_demo_scope, ) from lineageweave.http_client import HttpClientError +from lineageweave.observability import configure_telemetry _POST_READ = "post_read" _POST_ADMIN = "post_admin" @@ -195,6 +196,7 @@ async def lifespan(app: FastAPI): """Open one asyncpg pool and one Valkey client for the process, and close both on shutdown.""" + configure_telemetry("lineageweave") settings = load_settings() app.state.pool = await create_pool(settings.database_url) app.state.valkey = create_valkey_client(settings.valkey_url) diff --git a/backend/app/post_content_queue.py b/backend/app/post_content_queue.py index dae640240..3b4c0182a 100644 --- a/backend/app/post_content_queue.py +++ b/backend/app/post_content_queue.py @@ -10,6 +10,8 @@ import asyncpg import redis.asyncio as redis +from lineageweave.observability import traced + POST_CONTENT_STREAM_KEY = "post-content-ingestion" QUEUED = "post_content_ingestion_queued" RUNNING = "post_content_ingestion_running" @@ -126,15 +128,19 @@ async def publish_post_content_event( if client is None: return None try: - entry_id = await client.xadd( - POST_CONTENT_STREAM_KEY, - post_content_stream_fields( - post_id=post_id, - source_body_digest=source_body_digest, - ), - maxlen=1000, - approximate=True, - ) + with traced( + "lineageweave.valkey.post_content_xadd", + {"db.system": "redis", "db.operation.name": "xadd", "lineageweave.stream.kind": "post_content"}, + ): + entry_id = await client.xadd( + POST_CONTENT_STREAM_KEY, + post_content_stream_fields( + post_id=post_id, + source_body_digest=source_body_digest, + ), + maxlen=1000, + approximate=True, + ) except redis.RedisError: return None return str(entry_id) diff --git a/backend/app/post_content_worker.py b/backend/app/post_content_worker.py index 458b9021f..2077d90f7 100644 --- a/backend/app/post_content_worker.py +++ b/backend/app/post_content_worker.py @@ -16,6 +16,7 @@ from lineageweave.llm_context import build_post_llm_metadata, use_llm_metadata from lineageweave.post_content_normalization import normalize_post_body from lineageweave.post_content_persistence import persist_post_content +from lineageweave.observability import traced from lineageweave.post_structure import PostStructureClient from backend.app.config import load_settings @@ -41,7 +42,15 @@ async def _stream_tail(client: redis.Redis) -> str: """Start after historical wake-ups; the normalized ledger drives recovery.""" - rows = await client.xrevrange(POST_CONTENT_STREAM_KEY, count=1) + with traced( + "lineageweave.valkey.post_content_xrevrange", + { + "db.system": "redis", + "db.operation.name": "xrevrange", + "lineageweave.stream.kind": "post_content", + }, + ): + rows = await client.xrevrange(POST_CONTENT_STREAM_KEY, count=1) return str(rows[0][0]) if rows else "0-0" @@ -283,7 +292,15 @@ async def consume_post_content_stream_once( embedding_factory: Callable[[], EmbeddingClient], structure_factory: Callable[[], PostStructureClient], ) -> str: - batches = await client.xread({POST_CONTENT_STREAM_KEY: last_id}, count=10, block=1000) + with traced( + "lineageweave.valkey.post_content_xread", + { + "db.system": "redis", + "db.operation.name": "xread", + "lineageweave.stream.kind": "post_content", + }, + ): + batches = await client.xread({POST_CONTENT_STREAM_KEY: last_id}, count=10, block=1000) for _stream_name, entries in batches: for entry_id, fields in entries: post_id = str(fields.get("post_id", "")).strip() diff --git a/docker-compose.yml b/docker-compose.yml index 96ec0b89a..10d81800c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -117,6 +117,8 @@ services: # explicit bounded 8 MiB limit rather than an unbounded request size. CONTEXTUAL_ORCHESTRATOR_MAX_BODY_BYTES: ${CONTEXTUAL_ORCHESTRATOR_MAX_BODY_BYTES:-8388608} CONTEXTUAL_ORCHESTRATOR_ALLOWED_PROVIDER_HOSTS: ${CONTEXTUAL_ORCHESTRATOR_ALLOWED_PROVIDER_HOSTS:-host.docker.internal} + OTEL_SERVICE_NAME: ${OTEL_ORCHESTRATOR_SERVICE_NAME:-contextual-orchestrator} + OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-} command: ["python", "/app/start.py"] ports: - "${ORCHESTRATOR_PORT:-18000}:8000" @@ -155,6 +157,8 @@ services: OIDC_CLOCK_SKEW_SECONDS: ${OIDC_CLOCK_SKEW_SECONDS:-5} FRONTEND_ORIGINS: http://localhost:${FRONTEND_PORT:-15173} VALKEY_URL: redis://valkey:6379/0 + OTEL_SERVICE_NAME: ${OTEL_SERVICE_NAME:-lineageweave} + OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-} # Empty by default: every LLM/vision channel stays the Null client # (dropped, not faked). Set these to a running contextual-orchestrator # to turn the channels on. Provider credentials use LLM_GATEWAY_API_URL / diff --git a/docker/contextual-orchestrator/Dockerfile b/docker/contextual-orchestrator/Dockerfile index f4ef8eab7..0af60f58c 100644 --- a/docker/contextual-orchestrator/Dockerfile +++ b/docker/contextual-orchestrator/Dockerfile @@ -5,12 +5,16 @@ WORKDIR /app # Reuse the upstream implementation without copying it into LineageWeave. # Pin the runtime to a reviewed immutable upstream commit; model selection, # structured synthesis, and reasoning policy stay in contextual-orchestrator. -ADD https://github.com/ContextualWisdomLab/contextual-orchestrator/archive/7df051ac2b929e5910071ac1848d0447c5d6744e.tar.gz /tmp/contextual-orchestrator.tar.gz +ADD https://github.com/ContextualWisdomLab/contextual-orchestrator/archive/1a40e0f7ad10d1a24137d69d20e44fc9a5dcdd89.tar.gz /tmp/contextual-orchestrator.tar.gz RUN mkdir /tmp/contextual-orchestrator \ && tar -xzf /tmp/contextual-orchestrator.tar.gz --strip-components=1 -C /tmp/contextual-orchestrator \ && cp -R /tmp/contextual-orchestrator/contextual_orchestrator /app/contextual_orchestrator \ && cp -R /tmp/contextual-orchestrator/examples /app/examples \ && rm -rf /tmp/contextual-orchestrator /tmp/contextual-orchestrator.tar.gz \ + && python -m pip install --no-cache-dir \ + 'opentelemetry-api>=1.30.0' \ + 'opentelemetry-sdk>=1.30.0' \ + 'opentelemetry-exporter-otlp-proto-http>=1.30.0' \ && useradd --uid 10001 --no-create-home orchestrator COPY agents.json /app/agents.json diff --git a/docs/adr/0083-orchestrator-runtime-commit-pin.md b/docs/adr/0083-orchestrator-runtime-commit-pin.md index e5ea7083e..9bb6bd2b3 100644 --- a/docs/adr/0083-orchestrator-runtime-commit-pin.md +++ b/docs/adr/0083-orchestrator-runtime-commit-pin.md @@ -15,7 +15,7 @@ multi-agent. ## Decision `docker/contextual-orchestrator/Dockerfile` pins the downloaded archive to -commit `7df051ac2b929e5910071ac1848d0447c5d6744e`. The pin remains explicit +commit `1a40e0f7ad10d1a24137d69d20e44fc9a5dcdd89`. The pin remains explicit and immutable until the reviewed upstream change is superseded; it is not a moving `main` reference and it is not a LineageWeave monkey patch. diff --git a/docs/adr/0122-otel-session-observability.md b/docs/adr/0122-otel-session-observability.md new file mode 100644 index 000000000..cbd685e12 --- /dev/null +++ b/docs/adr/0122-otel-session-observability.md @@ -0,0 +1,54 @@ +# ADR 0122: Correlate product, orchestrator, and Valkey telemetry by post session + +## Status + +Accepted. + +## Context + +Post structure, VISION, OCR, embeddings, summaries, and queue work can run in +different processes. Existing ADR 0071 already defines a deterministic, +post-scoped session metadata value, but transport and queue failures were not +visible as one trace. The organization GRC repository owns the telemetry +control contract in [ADR 0009](https://github.com/ContextualWisdomLab/governance-risk-compliance/blob/develop/docs/adr/0009-opentelemetry-request-telemetry.md). + +## Decision + +1. LineageWeave uses the OpenTelemetry Python API and SDK and exports OTLP only + when OTEL_EXPORTER_OTLP_ENDPOINT is explicitly configured. The exporter + treats that value as a base URL and sends traces to its normalized + /v1/traces signal endpoint. The service resource name is lineageweave + unless the operator overrides it with the standard OTEL_SERVICE_NAME + variable. +2. Every contextual-orchestrator POST carries the existing + `lineageweave_post_session_id` as `X-LineageWeave-Session-Id`. The + orchestrator binds it to the request context and adds it to provider spans, + so chat, Responses, structured output, VISION, and embedding work for one + post can be investigated together. +3. LineageWeave emits bounded HTTP and Valkey operation spans. Valkey spans + identify the operation and logical stream kind, not the stream key, post + body, summary, actor, source identifiers, token, or provider response. +4. Failure logs contain operation, error type, status, and the bounded session + correlation only. They do not become a second evidence database. GRC may + consume aggregate control evidence and OTLP-derived SLO signals through its + existing contracts; LineageWeave does not copy GRC tables or credentials. +5. No ad hoc session table is introduced. The existing normalized post-scoped + session metadata remains the source of correlation. + +## Consequences + +Operators can follow a slow or failed post-content job from the LineageWeave +HTTP client through contextual-orchestrator and Valkey without exposing source +content. An OTLP collector is a deployment concern, not a local default, so a +developer stack remains usable without a telemetry backend. Raw session IDs +remain correlation data and must not be used as tenant, actor, or evidence +labels in GRC dashboards. + +## References + +OpenTelemetry Authors. (n.d.). *Manual instrumentation with OpenTelemetry +Python*. Retrieved August 21, 2026, from +https://opentelemetry.io/docs/languages/python/instrumentation/ + +OpenTelemetry Authors. (n.d.). *Service semantic conventions*. Retrieved +August 21, 2026, from https://opentelemetry.io/docs/specs/semconv/registry/attributes/service/ diff --git a/docs/doctoring/OPENTELEMETRY_REFERENCES.md b/docs/doctoring/OPENTELEMETRY_REFERENCES.md new file mode 100644 index 000000000..4cec4d416 --- /dev/null +++ b/docs/doctoring/OPENTELEMETRY_REFERENCES.md @@ -0,0 +1,27 @@ +# OpenTelemetry references and implementation traceability + +## Normative references + +- OpenTelemetry Authors. (n.d.). *Manual instrumentation with OpenTelemetry + Python*. Retrieved August 21, 2026, from + https://opentelemetry.io/docs/languages/python/instrumentation/ +- OpenTelemetry Authors. (n.d.). *Service semantic conventions*. Retrieved + August 21, 2026, from + https://opentelemetry.io/docs/specs/semconv/registry/attributes/service/ +- ContextualWisdomLab governance-risk-compliance. (2026). *ADR 0009: + Emit bounded OpenTelemetry request telemetry*. Retrieved August 21, 2026, + from https://github.com/ContextualWisdomLab/governance-risk-compliance/blob/develop/docs/adr/0009-opentelemetry-request-telemetry.md + +## Implementation mapping + +| Concern | Implementation | Evidence boundary | +| --- | --- | --- | +| Service resource | `OTEL_SERVICE_NAME`, default `lineageweave` | One logical service name per deployment | +| Post correlation | Existing ADR 0071 session metadata plus `X-LineageWeave-Session-Id` | Correlation only; not identity or authorization | +| LLM/VISION/embedding transport | `lineageweave.http_client.post_json` | Method, peer, bounded path, status; no body or credential | +| Valkey queue | `backend/app/*worker.py` and stream producers | Operation and logical stream kind; no stream key or event content | +| Export | OTEL_EXPORTER_OTLP_ENDPOINT | Disabled by default; base URL normalized to /v1/traces | + +The GRC repository remains the organization control and evidence owner. This +repository emits operational signals and does not copy GRC tables or persist +provider credentials. diff --git a/lineageweave/http_client.py b/lineageweave/http_client.py index 389b29f3e..3d6813334 100644 --- a/lineageweave/http_client.py +++ b/lineageweave/http_client.py @@ -20,6 +20,7 @@ import certifi from .llm_context import current_llm_metadata +from .observability import current_session_id, inject_trace_context, traced # Some interpreter distributions don't reliably inherit the OS trust store. # Pointing at certifi keeps full chain validation without weakening TLS. @@ -123,14 +124,31 @@ def post_json( request_payload["metadata"] = {**existing_metadata, **request_metadata} else: raise ValueError("metadata must be an object") - status, raw = _request( - "POST", - url, - body=json.dumps(request_payload).encode("utf-8"), - headers={"content-type": "application/json", **headers}, - timeout=timeout, - ) - hostname = urlparse(url).hostname or url + parsed = urlparse(url) + hostname = parsed.hostname or url + request_headers = {"content-type": "application/json", **headers} + session_id = current_session_id() + if session_id: + request_headers["x-lineageweave-session-id"] = session_id + with traced( + "lineageweave.http.post_json", + { + "http.request.method": "POST", + "server.address": hostname, + "url.path": parsed.path or "/", + "service.peer.name": "contextual-orchestrator", + }, + ) as span: + inject_trace_context(request_headers) + status, raw = _request( + "POST", + url, + body=json.dumps(request_payload).encode("utf-8"), + headers=request_headers, + timeout=timeout, + ) + if span is not None: + span.set_attribute("http.response.status_code", status) if status >= 400: raise HttpClientError(f"HTTP {status} from {hostname}") return _decode_json_object(raw, hostname) diff --git a/lineageweave/observability.py b/lineageweave/observability.py new file mode 100644 index 000000000..e7734f4cf --- /dev/null +++ b/lineageweave/observability.py @@ -0,0 +1,135 @@ +"""Bounded OpenTelemetry spans for product and infrastructure operations. + +The application emits useful correlation without placing post bodies, provider +credentials, actor identifiers, or arbitrary request paths in telemetry. +Export is opt-in through the standard OTLP environment variables. +""" + +from __future__ import annotations + +import logging +import os +from collections.abc import Iterator, Mapping +from contextlib import contextmanager +from typing import Any + +try: + from opentelemetry import trace + from opentelemetry.propagate import inject as _otel_inject + from opentelemetry.trace import Status, StatusCode +except ImportError: # pragma: no cover - dependency is declared by the project + trace = None # type: ignore[assignment] + _otel_inject = None + Status = None # type: ignore[assignment,misc] + StatusCode = None # type: ignore[assignment,misc] + +_LOGGER = logging.getLogger(__name__) +_CONFIGURED = False +_TRACER_NAME = "lineageweave" + + +def _otlp_trace_endpoint(endpoint: str) -> str: + """Turn an OTLP base endpoint into the explicit HTTP traces endpoint.""" + normalized = endpoint.rstrip("/") + if normalized.casefold().endswith("/v1/traces"): + return normalized + return f"{normalized}/v1/traces" + + +def current_session_id() -> str | None: + """Return the current post-scoped session without exposing other metadata.""" + from .llm_context import current_llm_metadata + + metadata = current_llm_metadata() or {} + value = metadata.get("lineageweave_post_session_id") or metadata.get("session_id") + return value if isinstance(value, str) and value else None + + +def inject_trace_context(carrier: dict[str, str]) -> None: + """Inject the active W3C trace context without adding request content.""" + if _otel_inject is not None: + _otel_inject(carrier) + + +def _safe_attributes( + attributes: Mapping[str, Any] | None, +) -> dict[str, str | int | float | bool]: + """Keep telemetry attributes scalar, bounded, and explicitly non-content.""" + result: dict[str, str | int | float | bool] = {} + for key, value in (attributes or {}).items(): + if ( + not isinstance(key, str) + or not key + or isinstance(value, (dict, list, tuple, set)) + ): + continue + if isinstance(value, str): + result[key] = value[:256] + elif isinstance(value, (bool, int, float)): + result[key] = value + session_id = current_session_id() + if session_id: + result.setdefault("lineageweave.session_id", session_id) + return result + + +def configure_telemetry(service_name: str = "lineageweave") -> None: + """Configure one OTLP trace provider when an operator supplied an endpoint.""" + global _CONFIGURED + if _CONFIGURED or os.getenv("OTEL_SDK_DISABLED", "").lower() == "true": + return + _CONFIGURED = True + endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "").strip() + if trace is None or not endpoint: + return + try: + from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( + OTLPSpanExporter, + ) + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + except ImportError: # pragma: no cover - guarded by the runtime extra + _LOGGER.warning("OpenTelemetry SDK/exporter is unavailable") + return + + resource = Resource.create({ + "service.name": os.getenv("OTEL_SERVICE_NAME", service_name), + "service.namespace": "contextualwisdomlab", + }) + provider = TracerProvider(resource=resource) + provider.add_span_processor( + BatchSpanProcessor( + OTLPSpanExporter(endpoint=_otlp_trace_endpoint(endpoint)) + ) + ) + trace.set_tracer_provider(provider) + + +@contextmanager +def traced( + name: str, + attributes: Mapping[str, Any] | None = None, +) -> Iterator[Any]: + """Create a span and a prompt-safe log event for one bounded operation.""" + if trace is None: # pragma: no cover - dependency is declared by the project + yield None + return + tracer = trace.get_tracer(_TRACER_NAME) + with tracer.start_as_current_span(name) as span: + safe = _safe_attributes(attributes) + for key, value in safe.items(): + span.set_attribute(key, value) + try: + yield span + except Exception as exc: + if Status is not None and StatusCode is not None: + span.record_exception(exc) + span.set_status(Status(StatusCode.ERROR)) + _LOGGER.warning( + "telemetry.operation_failed operation=%s error_type=%s session_id=%s", + name, + type(exc).__name__, + safe.get("lineageweave.session_id", ""), + ) + raise diff --git a/pyproject.toml b/pyproject.toml index cb4be2916..ed219f182 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,9 @@ dependencies = [ # docs/ontology/lineageweave-kg.ttl and the standards-complete PROV-O # support profile (ADR 0011). Pure Python, no Rust/C toolchain. "rdflib>=7.0.0", + "opentelemetry-api>=1.30.0", + "opentelemetry-sdk>=1.30.0", + "opentelemetry-exporter-otlp-proto-http>=1.30.0", ] [build-system] diff --git a/tests/test_observability.py b/tests/test_observability.py new file mode 100644 index 000000000..e67c2fc38 --- /dev/null +++ b/tests/test_observability.py @@ -0,0 +1,58 @@ +"""Tests for prompt-safe session propagation and tracing boundaries.""" + +from lineageweave import http_client +from lineageweave.llm_context import use_llm_metadata +from lineageweave.observability import ( + _otlp_trace_endpoint, + current_session_id, + traced, +) + + +def test_post_json_sends_post_session_header(monkeypatch): + """One post session reaches the orchestrator as a transport header.""" + captured = {} + + def fake_request(method, url, *, body, headers, timeout): + captured.update(method=method, headers=headers) + return 200, b"{}" + + def fake_inject(headers): + headers["traceparent"] = "00-11111111111111111111111111111111-2222222222222222-01" + + monkeypatch.setattr(http_client, "_request", fake_request) + monkeypatch.setattr(http_client, "inject_trace_context", fake_inject) + with use_llm_metadata({"lineageweave_post_session_id": "post-session-1"}): + http_client.post_json( + "https://orchestrator.example/v1/chat/completions", + {}, + headers={}, + timeout=1, + ) + + assert captured["method"] == "POST" + assert captured["headers"]["x-lineageweave-session-id"] == "post-session-1" + assert captured["headers"]["traceparent"].startswith("00-1111") + + +def test_current_session_id_reads_existing_context(): + """Telemetry reuses the existing normalized LLM context, not a new store.""" + with use_llm_metadata({"lineageweave_post_session_id": "post-session-2"}): + assert current_session_id() == "post-session-2" + + +def test_traced_rethrows_provider_errors(): + """Observability never converts a failed provider operation into success.""" + try: + with traced("lineageweave.test.failure"): + raise RuntimeError("provider failure") + except RuntimeError as exc: + assert str(exc) == "provider failure" + else: # pragma: no cover + raise AssertionError("traced must preserve operation failures") + + +def test_otlp_base_endpoint_gets_trace_signal_path(): + """A configured collector base URL receives the HTTP traces signal path.""" + assert _otlp_trace_endpoint("http://collector:4318") == "http://collector:4318/v1/traces" + assert _otlp_trace_endpoint("http://collector:4318/v1/traces/") == "http://collector:4318/v1/traces" diff --git a/tests/test_post_content_worker.py b/tests/test_post_content_worker.py index dddace990..8ddb16628 100644 --- a/tests/test_post_content_worker.py +++ b/tests/test_post_content_worker.py @@ -229,7 +229,13 @@ async def persist(*_args, **_kwargs): ) updates = [args for query, args in connection.executed if "set status_code" in query] - assert any(args[1] == QUEUED and args[6] == "post_content_ingestion_failed" for args in updates) + assert any( + args[1] == QUEUED + and args[6] == "post_content_ingestion_failed" + and args[7] == post_content_worker._UNEXPECTED_FAILURE_DETAIL + for args in updates + ) + assert all("provider timeout" not in str(args) for args in updates) def test_failure_at_attempt_limit_is_terminal_and_visible() -> None: diff --git a/uv.lock b/uv.lock index 10bcf9ff1..571f1a9c8 100644 --- a/uv.lock +++ b/uv.lock @@ -167,6 +167,137 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/aa/29/35e016098c814cd93de9cd320c66b5bfba14dc6ecedd3cb518fa7c408c69/cffi-2.1.1-cp315-cp315t-win_arm64.whl", hash = "sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692", size = 186360, upload-time = "2026-08-03T21:21:13.636Z" }, ] +[[package]] +name = "charset-normalizer" +version = "3.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e5/3f/143b048436775b0f76ac3eec145c019e8173ccc2885c8f20319b996d5e83/charset_normalizer-3.5.1.tar.gz", hash = "sha256:6117b84ea48435e5356dc737f5121485c30920ba43375fa7b434fd753df0eac3", size = 171764, upload-time = "2026-08-15T08:20:44.807Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/30/27/78873dc8b6a56357517b74b6bb9568b80450e7bb4f6ef7e3fa9d22aa0bd7/charset_normalizer-3.5.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:5b6d1386bf0096d26d3a863dc0a487a5b4eb9aa93cf5ba69683d29dde6b9d60f", size = 344456, upload-time = "2026-08-15T08:17:10.072Z" }, + { url = "https://files.pythonhosted.org/packages/9a/4c/be49ada26b1f0232d57aa89bbebf997a5cc2332a5616b6eca26ff680044d/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4582c27e8c889d64811987b5967fbd3ae0c823fe1fd933b543d55ac20bb475fa", size = 238530, upload-time = "2026-08-15T08:17:11.563Z" }, + { url = "https://files.pythonhosted.org/packages/76/84/6f1290fa07ae6978d3960caa3eb1b8019bf9284ab7c2297b00c099ef4250/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1d1c7a53a6c2103925cdd6d7229f8c567379f211c869793df679f2e9f738c369", size = 230200, upload-time = "2026-08-15T08:17:12.919Z" }, + { url = "https://files.pythonhosted.org/packages/e7/a0/47b18adeed31c8f16ba9700f32c1b18594cfa09f47eb672a488c273c22bf/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e6621fb2a4988d6e53eedc455e5903e2679f3967b8acb3d639f1b63c14a2e893", size = 262222, upload-time = "2026-08-15T08:17:14.571Z" }, + { url = "https://files.pythonhosted.org/packages/38/fe/341861ac118dae06f3ec0eb487488af52128f2ef2faf0b11003944d22259/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7c0c10730342b0c9b35dd1d619beb8214e520bd96a1f870f452680b238aab3e0", size = 258951, upload-time = "2026-08-15T08:17:16.158Z" }, + { url = "https://files.pythonhosted.org/packages/6f/89/bb5108dc6c3651dca963f2b0a3ba19bbcb370c94e1b6d3e0e844a58e6dca/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b9af956078716df40d985fb0dfeb2c2120c5ca92ba4ff4b388acfd01cdc14d08", size = 248801, upload-time = "2026-08-15T08:17:17.683Z" }, + { url = "https://files.pythonhosted.org/packages/b1/ba/ef83ae3aca816393decfa3530976f38a79812d707b80b580ac33b83f9877/charset_normalizer-3.5.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f9f8405c2c758532c74fed975dbee57be1f31a6e865c031870c79a6ed3212ada", size = 244070, upload-time = "2026-08-15T08:17:19.191Z" }, + { url = "https://files.pythonhosted.org/packages/f6/0b/c5292a2462d69b7378ea89793bbb5b2b6fcf6f7dd6d1667f9619094ad553/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:96fef3e886d6a9874b14f27fc193fbdc69d5d8035783d86aa4e1cea594e695f9", size = 240110, upload-time = "2026-08-15T08:17:20.547Z" }, + { url = "https://files.pythonhosted.org/packages/46/22/111e5be3b740d5c2a5bfcedb3d237b6591e5c2e82ae9d6ffcb121fe0909c/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5d8531a6569d025f68e2321e7638fb7978f23db58e5f69f56913837aae03816e", size = 232836, upload-time = "2026-08-15T08:17:21.895Z" }, + { url = "https://files.pythonhosted.org/packages/f9/d2/d2aad6fe0dbb44b194bf3becb60f5a0ac48446ade999a47fe7bb41eb09a7/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:aae2ee51122d3ae968a3837d97dc24a0aeebb0dea23694422cd172bd30017cd6", size = 262712, upload-time = "2026-08-15T08:17:23.727Z" }, + { url = "https://files.pythonhosted.org/packages/35/5a/337e4663a5eae6de99db940ee8066d4145caafb61327db62deda15313cce/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:7235dc28fc6dd9d832ac7c7bce95367dedb85929f17368a0c2bee1e080b9acbf", size = 242977, upload-time = "2026-08-15T08:17:25.157Z" }, + { url = "https://files.pythonhosted.org/packages/ca/85/f82f8a92e31c7519410e2e1afdc630f28ec47490ce2c09a11c1a43cbb459/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:4abdc5f9ad448c1ecbfae2974b820535d6bc6e7eef63babbab3d81cf46968c71", size = 260207, upload-time = "2026-08-15T08:17:26.602Z" }, + { url = "https://files.pythonhosted.org/packages/b7/52/643d11ffd60e9ac2fd1fb87e167a19285b9eefeff4a40e63c87cbfbeab36/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ba501e667c17d8411f98e67a022d9604ef179aff0e459b7e292c796837c13573", size = 250562, upload-time = "2026-08-15T08:17:27.971Z" }, + { url = "https://files.pythonhosted.org/packages/62/16/46556278c2168d12df9da7fede5dc6fc70e60301b26a82bbeec238c9cfe3/charset_normalizer-3.5.1-cp312-cp312-win32.whl", hash = "sha256:cfa1c0cc3a8f9f53f1243a5a99ac36fd003880199383b37672e86ddda9cb07e2", size = 178507, upload-time = "2026-08-15T08:17:29.277Z" }, + { url = "https://files.pythonhosted.org/packages/9d/7a/4c6c298171e6b3e745633180ff59350fc0ca0db1ffd28df1e369e0579f71/charset_normalizer-3.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:3617ac3cfd8b9888f145ad89dd6e692285834b0201c6074a5eeaad3fd4d668c2", size = 200551, upload-time = "2026-08-15T08:17:30.668Z" }, + { url = "https://files.pythonhosted.org/packages/cd/d7/eb95a042f0dd22e304b0b6472b154f3546a1a039a9ee89ccb2a7f61591fc/charset_normalizer-3.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:88e85ab89cb822c1e635f51d6d32e488f94e002e70e2f492bdb8b945543f345a", size = 180700, upload-time = "2026-08-15T08:17:32.028Z" }, + { url = "https://files.pythonhosted.org/packages/bc/61/2cb6ad133dbbb449fa2d37ccae973232f4827e799af258d15e589a3d1e9e/charset_normalizer-3.5.1-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:4f298bdadb8f0b9e5672877f647d1be9373ef5320c9e2f049795e26cad28b6a9", size = 211584, upload-time = "2026-08-15T08:17:33.597Z" }, + { url = "https://files.pythonhosted.org/packages/18/57/a305c968be1ca13f3dd1b32f445877e97addf55d80b65c7cb35fac82b777/charset_normalizer-3.5.1-cp313-cp313-android_24_x86_64.whl", hash = "sha256:88ca277405c2d3b71c4e1c2ee0e7966e807bcba86a69d11e19ba199d18ae4491", size = 223359, upload-time = "2026-08-15T08:17:35.022Z" }, + { url = "https://files.pythonhosted.org/packages/09/0a/d3646670292ce8d8f8cc11ac067d44885e697a5591f57a9221128da5e7b3/charset_normalizer-3.5.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:9362dd90aa7dab48c0054a21187791ccf05473f7dba5d92b8033ae62164675e7", size = 194464, upload-time = "2026-08-15T08:17:36.452Z" }, + { url = "https://files.pythonhosted.org/packages/de/93/d51ec556e01042fed6f993ea859311bc7917b466684182fbbceb6ca24762/charset_normalizer-3.5.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:977cdbd483a9cff38179bea4fd754289a6f2195c7abd414aba85410b3e66cc5e", size = 197676, upload-time = "2026-08-15T08:17:37.819Z" }, + { url = "https://files.pythonhosted.org/packages/a4/a0/562247944386f7d4ef94467e84876600cc1e0f1b93239aaa9213d2bc3cbd/charset_normalizer-3.5.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e90251c0c7bdd54a100a0dce3c07b7e637278c93af29dbf78ebb89a58c4bac7d", size = 340473, upload-time = "2026-08-15T08:17:39.303Z" }, + { url = "https://files.pythonhosted.org/packages/31/e7/1d994be1b93d41e9502b8b0460eaa88a1dd8df335df415db87d6c3e91ab2/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:94d78ecec2605a8d0398b0f365d5f12a63248438516f5dac536a5eff7337df4a", size = 240156, upload-time = "2026-08-15T08:17:40.66Z" }, + { url = "https://files.pythonhosted.org/packages/09/53/27923ce5cc6cbccb832037b27dca98882d9c53e9b69e866bbbef4aae7fc8/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d59b75732e9b6f27388e10c14b0259cc5f2e48c78627d185e6a177b58ad3cffe", size = 228246, upload-time = "2026-08-15T08:17:42.003Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/5a97e84d63af1d55c07439cb80e56d99a8efb4295700eb4e18c0d1615d2c/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0d929fc574b4d6fd9e7c0f5c2ede8716a41911923aa7fa5fce38e0818aa4a1ac", size = 263660, upload-time = "2026-08-15T08:17:43.627Z" }, + { url = "https://files.pythonhosted.org/packages/7a/c2/071575791dcc88316c0a9a65ce38897a82e4cfe4a325f0f7fe1b1ac47bcf/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:394fea06235c8543390050ed5f529187074b029fb027213f6c46ac11ab5d950e", size = 260354, upload-time = "2026-08-15T08:17:45.094Z" }, + { url = "https://files.pythonhosted.org/packages/fb/af/63240b0c0248c075c2535a1f1bd992821d8251b9f173abc13329661d09e4/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:62b55f6722735a6c472f88361cde6640608773d9443cebdbb51abf436a1fcdd3", size = 250638, upload-time = "2026-08-15T08:17:46.496Z" }, + { url = "https://files.pythonhosted.org/packages/4d/66/70dfad64f15be09c15ccfee81330a7e515895dbe296dd23114e9a231268a/charset_normalizer-3.5.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fa48b1b63d639f9483e0633e092f5851e2348c352f1f9bb6c8182f87884ef876", size = 244583, upload-time = "2026-08-15T08:17:47.963Z" }, + { url = "https://files.pythonhosted.org/packages/c0/24/ef36367d38b9ddd4bccbf72888c342e8de1f5ae506fa0b2dcf970e2732a1/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c71fb0d56c920c269cd3e2e3fe7c610e3f1fdb21a6ce60efa6430ff63676cea6", size = 242038, upload-time = "2026-08-15T08:17:49.481Z" }, + { url = "https://files.pythonhosted.org/packages/db/ab/55e683ba0fff2e43adafc10daa3001eac90fdaa419a97227d5a7067eedde/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:485a0d363cafefcd2538a73c7c838daa2035f09b2c9f9b5e3133f80c6aeb84c2", size = 233677, upload-time = "2026-08-15T08:17:50.845Z" }, + { url = "https://files.pythonhosted.org/packages/bd/67/0f40eaf8d1b6e7cf15e82382a2965efaca787fc1c2794b7021d37aaf5036/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0ea61a470e070686aa30892fed79e297d2c8d0ab46b8bcdf027d38c51da591", size = 264491, upload-time = "2026-08-15T08:17:52.61Z" }, + { url = "https://files.pythonhosted.org/packages/5c/64/12b4c2a11ee8df4fcc518c78b0d93e3a92bd3d5253d1617ce74ff0e8c7ef/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:90b7481fb62fbe172c558bc6fd1c4c98d82004a54a7551f20e11ac9bf0b8708c", size = 245196, upload-time = "2026-08-15T08:17:54.023Z" }, + { url = "https://files.pythonhosted.org/packages/37/2e/651d910af6d0fba325eee1cda37ec5443462ed25360e666c144166eb6091/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:35fe081843b35aad20ffeccec3eeffbe637b15d14f3fb22cc1b59cd8ec17e93c", size = 261660, upload-time = "2026-08-15T08:17:55.491Z" }, + { url = "https://files.pythonhosted.org/packages/90/c6/b09e05e6db7f64338e0dc067c79577b1138da86c1e38369096851d96be88/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fd0350afdc3aabd5576f60ea109228bd5538139713c7b094c5cd27c73a98bc6f", size = 252618, upload-time = "2026-08-15T08:17:57.025Z" }, + { url = "https://files.pythonhosted.org/packages/76/4e/362d4f9fdcdf5556fb2aa3ce7d4a58ebce03ed1ff03aa1d9aca8d02f13f3/charset_normalizer-3.5.1-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:9d9a0dc7cbe9bec24c3f767c9122c41fe5a1bc43f47cd099d00d393e09769de4", size = 140362, upload-time = "2026-08-15T08:17:58.425Z" }, + { url = "https://files.pythonhosted.org/packages/b4/d4/703be739b26acce318bd29eb3b25b7209e1b1f527f9eae3d1f1f01fdde2b/charset_normalizer-3.5.1-cp313-cp313-win32.whl", hash = "sha256:d63600d620ad0064c3a748b950ac5ea38a80190e5498532efefa4b7b3f1da1f3", size = 177755, upload-time = "2026-08-15T08:18:00.037Z" }, + { url = "https://files.pythonhosted.org/packages/8a/33/56d97ade41c8db611e727168c52ae46c9224c362ec28d4b65d7e9869e8da/charset_normalizer-3.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:aea996a6aba25260827c9ea511d1addfde2da9eb686ac961838509086188b7e6", size = 199295, upload-time = "2026-08-15T08:18:01.506Z" }, + { url = "https://files.pythonhosted.org/packages/5b/75/5b20dd1e6573a01a08158fe104104fa2c8abf941745596954185726cd46c/charset_normalizer-3.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:fd0a274c0e5f9a21565cd9d3dd749b61f96b7aa1e20a93aa1ba4029518f2e5c0", size = 179856, upload-time = "2026-08-15T08:18:02.929Z" }, + { url = "https://files.pythonhosted.org/packages/29/cd/2b812ce5e888f1ce69a5350281e58aab07ae64a958ecae8912f30865718e/charset_normalizer-3.5.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:774d157f112367ff4abd29019f38f023c24e00e56edc7829c20e358a5a913ad8", size = 212318, upload-time = "2026-08-15T08:18:04.403Z" }, + { url = "https://files.pythonhosted.org/packages/9e/4a/a6ee107430768a5334e6d63f31f148a04a1a491ef161a1ac9415a73f2fa8/charset_normalizer-3.5.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:26422d45fd13551cf564c58932f7d72b4f58b93b0fcf18c35ba6be12b46bb102", size = 224897, upload-time = "2026-08-15T08:18:05.997Z" }, + { url = "https://files.pythonhosted.org/packages/c3/d9/35ae3f64f29d0179c35c3baefe575904df2913dde519129c7f75995a2b1d/charset_normalizer-3.5.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:09a7bba9f739468c8e78c36a75c33768e53cb1959fc638f510454c14683f00d5", size = 194848, upload-time = "2026-08-15T08:18:07.397Z" }, + { url = "https://files.pythonhosted.org/packages/74/76/f2fc7380f056cc273a53af37f50d08ad54b2c59f61078f31432edcf1c2bd/charset_normalizer-3.5.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4c9548dc78002099910abaebc0a72ac58b7d30931869e0351c09b507dff4ece3", size = 198163, upload-time = "2026-08-15T08:18:08.989Z" }, + { url = "https://files.pythonhosted.org/packages/e9/40/095ce62fa078483cccc1fa2b36e6bc9580b85422a20ee9f925341c50e44f/charset_normalizer-3.5.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c428c6c31eb5f4277d7f8eccaf767fbd548ddd5ce3c8b4f4cbbfab3d96b5904c", size = 341823, upload-time = "2026-08-15T08:18:10.458Z" }, + { url = "https://files.pythonhosted.org/packages/f1/5a/0e58b1c04a1596e0256f407274a92d5fb2ee21324409d1fab1da48a65b5b/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2f06b7eae9dbe77fe1d644ca244dad508de8d302870a43f3c559b521270938a0", size = 242458, upload-time = "2026-08-15T08:18:11.989Z" }, + { url = "https://files.pythonhosted.org/packages/22/95/b4618ce912e6db0b1aae89ba788e38e8a7eba0f3025cc66e8c0699f977b2/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6b7430cf5728e68f6c462254009a6ef4086e1bea43cf2f57aa9c55fb4f50ff96", size = 226717, upload-time = "2026-08-15T08:18:13.401Z" }, + { url = "https://files.pythonhosted.org/packages/8a/76/c681192bbda3d55356db5dadd64381d5202b37c6b598fcda5282e88b5d3d/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ab743e9bc90c1f73552ec33e10e3331315acd2c397b36065b591b0181de533cc", size = 266111, upload-time = "2026-08-15T08:18:14.961Z" }, + { url = "https://files.pythonhosted.org/packages/88/be/55127bfca72c0cff6c022488d140d7c5b04c771e3b72e9bdb4836d54979d/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f6f7deae3feb4edfa2efaf7c574fe88cbf055038a6abdb40188e4fff66d5699f", size = 263128, upload-time = "2026-08-15T08:18:16.515Z" }, + { url = "https://files.pythonhosted.org/packages/e0/91/39c3af510b0aa32bbda03374259200f28430febfd1bf5e511fe765282ce5/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:15f024313246a4ed976c60f440bb8d257815513a681d212ff74fd46f7d715a90", size = 251240, upload-time = "2026-08-15T08:18:18.127Z" }, + { url = "https://files.pythonhosted.org/packages/1c/a5/cbe418bbc6ecdfc3e05a0116002897c4b403a5e838d697e64c78e9f0190d/charset_normalizer-3.5.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:823f82903d189af463d7df250ef1f7f696f3cee08cc8d91deb565e8d425f6506", size = 245282, upload-time = "2026-08-15T08:18:19.625Z" }, + { url = "https://files.pythonhosted.org/packages/cc/a4/689bb42e8e7cd492f3cb64907c6bc00ad247ec9a3628cd3f8eed126e8ae1/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:01e93745f7f219b703b60ba7afead36cfc4242782be5af484673fc500df12da5", size = 244597, upload-time = "2026-08-15T08:18:21.121Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ce/9962938e179cf9f699d3f1e7b3114b5d7642dee6a893745229f9dd04f274/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:329fc3ccb63ad22d867d84c2adea759a64079a37ba4a343433b02c7a2816871e", size = 231376, upload-time = "2026-08-15T08:18:22.57Z" }, + { url = "https://files.pythonhosted.org/packages/85/54/46000450ada53bd9eac5429a2c8c54cd2d9b39c0c255f229aea9af0948a5/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:bb57753e36e4855b8ca375069482250a6246372331a3e4f3407eaebb007443f5", size = 266715, upload-time = "2026-08-15T08:18:24.235Z" }, + { url = "https://files.pythonhosted.org/packages/3d/bb/618749d70f792b44252a777bf89bfb86823b9bbc1ea13fe8ce759b07f38a/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fce8cbd4997efeb450bd298b54f755dcdff18d496f7a5ddbb4867c6d7c88fdc3", size = 245848, upload-time = "2026-08-15T08:18:25.726Z" }, + { url = "https://files.pythonhosted.org/packages/7e/3f/ffb64458527c7668031d5eb095d978de561958dc9f5b53f8e488a533e603/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:6c9cdde8becb25a7fde49924511aa2644d6f8081cc8df8e9452724303348d8e3", size = 264521, upload-time = "2026-08-15T08:18:27.193Z" }, + { url = "https://files.pythonhosted.org/packages/4f/ab/74a55fd803916a35ac461daf002708191aac19b546b80dc8cabfedc63d98/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9ac4444d8d4fd4c4bd08bf451ed3167aa9e7ec6cdb41b648794f1d1103652e36", size = 253054, upload-time = "2026-08-15T08:18:28.568Z" }, + { url = "https://files.pythonhosted.org/packages/a0/2a/6a9034b7d3c60b17499afb482df5878bf9fa20b50cc3887d5ef017a833db/charset_normalizer-3.5.1-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:f03ac127268b43ef4fe9e6ab6794a6794b49485a0cc0c1db79876d2f33f75bc7", size = 140580, upload-time = "2026-08-15T08:18:30.214Z" }, + { url = "https://files.pythonhosted.org/packages/f3/46/1d362e1a00d035d66b9869e1281eee115907f7e390a16a07824ab5737360/charset_normalizer-3.5.1-cp314-cp314-win32.whl", hash = "sha256:1f5883d77fd409a261abb5dc8ccbe335720d798b1de4abb3b1d47ccbbc76b53b", size = 180325, upload-time = "2026-08-15T08:18:31.877Z" }, + { url = "https://files.pythonhosted.org/packages/7a/7c/4938c329b6a9d446f6a59aa2092ff7118f274209b5ed0e26893d1d30a63c/charset_normalizer-3.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:c658c50ac0c98cd755a2dd50b7977d3bca7df401dcc47fbdfa87db53ef7d4e8b", size = 204175, upload-time = "2026-08-15T08:18:33.466Z" }, + { url = "https://files.pythonhosted.org/packages/ac/33/eeb384dbd8dec570661354592f4f2e1b2fcc92585624d146a000caf53841/charset_normalizer-3.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:4bea7f8ebe90bbd7f0e4a2de42ca6924ba23e3e76418c408ff82f1d46fabd687", size = 184123, upload-time = "2026-08-15T08:18:34.913Z" }, + { url = "https://files.pythonhosted.org/packages/1c/6c/c73fa9d5a85f6ab05395de61c5f6984e0a9ff40bb5ff888d46dff02526c6/charset_normalizer-3.5.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:fbc597639158fd7c14d55e808718848319540f51b0e6746e3eefa59723a4a348", size = 381682, upload-time = "2026-08-15T08:18:36.349Z" }, + { url = "https://files.pythonhosted.org/packages/30/c7/63565f860921457feba93bae6c86fb7746deb4cffeed2f375cb845318146/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e71c909f353863b2b89c83de2ebed71ea6d0df8a6ef65a128193c5e650766bef", size = 240826, upload-time = "2026-08-15T08:18:37.887Z" }, + { url = "https://files.pythonhosted.org/packages/06/ae/7ae8807410dfa33f8e6f1715740adeaafa8a816cc4cb33508f54b1f7c896/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7ac76cf9afd34929d76eb7fcb63be476a4853d8a96f0dcf2d0db68a0cbdf9885", size = 227861, upload-time = "2026-08-15T08:18:39.315Z" }, + { url = "https://files.pythonhosted.org/packages/e9/a3/887c1642f0da26000b0e0652d91071113c0e72cea33952e225cf589f49a9/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a3a370082ce34d0612f421e15fe011c53bb1feff21a26d06ad4fb244dab5a375", size = 260758, upload-time = "2026-08-15T08:18:40.88Z" }, + { url = "https://files.pythonhosted.org/packages/3e/11/e6f5b9a3d0e55b0ef7505cd3765cdd48f22db89994c947b316f52f801fd8/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:256dd4d85d9e4dc595e2bc983c980e73f62ddeb3165c58b4c3dfe78c5c8548c1", size = 259950, upload-time = "2026-08-15T08:18:42.351Z" }, + { url = "https://files.pythonhosted.org/packages/1b/ee/e4e10a94d51cd1ee638aa7e00b65399e6b2a4e8376ab6d2eac9f95586671/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:58d4aa13a59c969dbfdf9e6a9560e242cbfd9e8a8f50c2747714df1a423adf65", size = 249329, upload-time = "2026-08-15T08:18:43.914Z" }, + { url = "https://files.pythonhosted.org/packages/c4/25/d5f4198819e6059735a84e8d0bfb72dc33976da67b97adcd3fb5a5e07ec6/charset_normalizer-3.5.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0c6dfb5ca6723eeed15aa8e564a014d69fcb8812f94eef11fe3631e0508199f5", size = 243137, upload-time = "2026-08-15T08:18:45.368Z" }, + { url = "https://files.pythonhosted.org/packages/a5/e9/e925ca7569cf9fb9701fd82503fee73eea5268fdb856bdd64947092d3daa/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c010f5581d9c612804cc59fcf7b524b707fbcb72828551237ab545bb5c7034af", size = 242820, upload-time = "2026-08-15T08:18:46.842Z" }, + { url = "https://files.pythonhosted.org/packages/34/17/672c251a888ed2aebcdd2fe830ad0104e25ff83c43f5c4f9c15e9fc6853c/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:52ec005752a56ae79547a05c0139ca2501a0c866390b6115008456b9f0e7cde1", size = 230504, upload-time = "2026-08-15T08:18:48.353Z" }, + { url = "https://files.pythonhosted.org/packages/3f/fc/f6a85abebd42ce4da2f1db0aa56cc6a0df1995e318b3875d14401b8381d1/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2bced4061f000f7187254a02ad3433ae17eaf991747ceea2f478422590a5bba9", size = 263087, upload-time = "2026-08-15T08:18:49.859Z" }, + { url = "https://files.pythonhosted.org/packages/98/66/7c42677e739ba66746b297e2046918d793078094dc239e1e72768cffccc6/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:9eea3ab2597a5e65fe65296e2d6a84570845a6b55532d90333d740d48bbc850a", size = 243269, upload-time = "2026-08-15T08:18:51.601Z" }, + { url = "https://files.pythonhosted.org/packages/de/d8/a50b79237f417af10f8c2a501ce8d1ca87829a22e69117891ca4ba20a69e/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:496846868fea80e479324862fa877f02411f2fd0f83b79ccee2607aa68b2a032", size = 258766, upload-time = "2026-08-15T08:18:53.23Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1d/0fc91aeaeb3c83b748f532399ce67cf84604b48297405d740000f7a9e786/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:85d5855daafc240cc045c026d7a15fd198a09b0fc8ff6f5ecbb5297b509cb11e", size = 250814, upload-time = "2026-08-15T08:18:54.768Z" }, + { url = "https://files.pythonhosted.org/packages/ae/10/3d8c777cf9024615295aa1b808324ad5b4a77855869c00824bad74ffaf8a/charset_normalizer-3.5.1-cp314-cp314t-win32.whl", hash = "sha256:58d3e12c88e0950bca850ae1f7c256055c097639c2edb9eb123af9807d8b15e4", size = 191074, upload-time = "2026-08-15T08:18:56.305Z" }, + { url = "https://files.pythonhosted.org/packages/4d/81/ae557d3c44d1a1d688696d60563413a0866a91b7ebc50f20df838be3d8c8/charset_normalizer-3.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:acaf604462bf330b0d07e7a07c1d6e4adac79e5fb13e9c5140590542cafacc00", size = 216476, upload-time = "2026-08-15T08:18:57.889Z" }, + { url = "https://files.pythonhosted.org/packages/27/e9/61c01fb8b804692569c036b3fc50495814502dcf13a60649c6055390b02c/charset_normalizer-3.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:fdb8a068947befafba9952162645dc2fecaeb400e64584829ed5e9b2fbe21a7f", size = 194115, upload-time = "2026-08-15T08:18:59.418Z" }, + { url = "https://files.pythonhosted.org/packages/4a/4e/8544831ef59d8f27ce92c80871380fdacc8076a8a56ed62f82e54f991333/charset_normalizer-3.5.1-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:9085f87b0e38a2b92b8923059b4e8789fe40d9279712d15dcc670048d77079af", size = 342048, upload-time = "2026-08-15T08:19:01.054Z" }, + { url = "https://files.pythonhosted.org/packages/7f/a6/e3b46852424246065355644f4fb6dbccc0239a42a2eee27ecfc8957f0bcd/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2679de311c7946dde5d3b6f44941844133ff5c7cb86099c0061ab1e8901c20a8", size = 242997, upload-time = "2026-08-15T08:19:02.492Z" }, + { url = "https://files.pythonhosted.org/packages/03/3b/0cc9a26777334ab2f2e3089b948bbf4e4fe72ea70b897715ef6415043ec8/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:baf3775a2635e5a11fbd5e4e64ee69c7e86875d224a5c72aca4c141064589a90", size = 237014, upload-time = "2026-08-15T08:19:03.943Z" }, + { url = "https://files.pythonhosted.org/packages/8c/c2/027335f0aa337a2a2e121bac1ad88c4f02ba6053ea0926802784f3db11af/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8ac8c94b6539074e0f40899301273ac8402b9b3e01c7b7ba269ff30340aaaf20", size = 266174, upload-time = "2026-08-15T08:19:05.598Z" }, + { url = "https://files.pythonhosted.org/packages/86/d3/e367787febe4e74769dec0f406f2c3c8d1b955fce5aee1fd0f94e8367a45/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8fe532b3c966d1fb794e0698e4589d0444017ae77fc0b31edea13c0e35bcc449", size = 263361, upload-time = "2026-08-15T08:19:07.251Z" }, + { url = "https://files.pythonhosted.org/packages/af/3d/391b193eb9f3e84b02f9314088c386debdc0debee843535aaea2e2c6715d/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5c84bec0ab5ae0c64bfe73a7d2adcb5ce73b467523fc27fd6a28ab2aa6cbe35a", size = 252143, upload-time = "2026-08-15T08:19:08.816Z" }, + { url = "https://files.pythonhosted.org/packages/2e/57/de221f1745a90d418199761967e2776bfe2c275a1194220985e8c1d37833/charset_normalizer-3.5.1-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:854066be00447fa8de2ccbbe893e2ffc4b123ef16d897af794c1e18bd4a714b0", size = 252086, upload-time = "2026-08-15T08:19:10.255Z" }, + { url = "https://files.pythonhosted.org/packages/c8/e3/d119f86a01f9331e8186175f24873b1d74a7ee9e2e4b4d68f9947dae5afd/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:21b82d8082f6f5e7f456ef0bd16323d08de1266efbfeb476e64b2a91d1471a4e", size = 245231, upload-time = "2026-08-15T08:19:11.807Z" }, + { url = "https://files.pythonhosted.org/packages/26/de/d8e48c135ae480879539cdb179c8d3b50c7879497d75dd899b5763b69cee/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:838648accb3a7fd9803fd45c87bce8509648eb0c11bc34e216141300977244f2", size = 241546, upload-time = "2026-08-15T08:19:13.416Z" }, + { url = "https://files.pythonhosted.org/packages/67/c4/217755fd1abc50d326c252922cd642002758095a81ff45010337b8b3ef65/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:195ce897c6153c0700078142cf8efe3e6454ca4cf4357499e4078dfd83396626", size = 267033, upload-time = "2026-08-15T08:19:14.981Z" }, + { url = "https://files.pythonhosted.org/packages/b8/d7/34d8e404e358d2adcc5a228c2134643af00104c8fb0bf525f3688d756f05/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:978eab16f55b4ab2c2a745be9a0a840bf8f09a7f227d9c76eb30214d078865a5", size = 252045, upload-time = "2026-08-15T08:19:16.618Z" }, + { url = "https://files.pythonhosted.org/packages/5e/fa/40414471acf0aa0692ca77305aa00e434fcd8288f0941c93c30e9a5f8f2f/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_s390x.whl", hash = "sha256:cc0329df4caaceb950d2f580b5ac716a377f7059624a0bafaeaf8a218c6ed774", size = 264866, upload-time = "2026-08-15T08:19:18.101Z" }, + { url = "https://files.pythonhosted.org/packages/32/90/fcc850bae791abd2e0c041847f13e270aa08692a79f3e00de6d2dce1cb50/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:687c9ca3035544b113bea2055e180af96fb63c0c476e22a9180f51925186e7b7", size = 253932, upload-time = "2026-08-15T08:19:19.734Z" }, + { url = "https://files.pythonhosted.org/packages/af/af/53afe99068b3c10b4cbae592a52ef72a7c92c0188440e83ee3a078fd8f75/charset_normalizer-3.5.1-cp315-cp315-win32.whl", hash = "sha256:706bfd38730a5ac7a365793269a00f4e988178cec121391f4248d84ad8c972e9", size = 180320, upload-time = "2026-08-15T08:19:21.37Z" }, + { url = "https://files.pythonhosted.org/packages/c9/bc/f46a132041b29e4a8779ed712d3df1bf112e94ca8de58b66d7ec2c0cf8b9/charset_normalizer-3.5.1-cp315-cp315-win_amd64.whl", hash = "sha256:92caef967d287a407085d61176fce4012b1dd62daed4eb6d5ceb26d3d2538712", size = 204174, upload-time = "2026-08-15T08:19:23.088Z" }, + { url = "https://files.pythonhosted.org/packages/a1/5d/9ed554480eda8e447b673648628fdc29574d23dbad01fe11837adedd1cae/charset_normalizer-3.5.1-cp315-cp315-win_arm64.whl", hash = "sha256:5fc45d653ea8c9a20479167e11d4a0f8cb2fa3470737ab6f9c827532313187b7", size = 184126, upload-time = "2026-08-15T08:19:24.471Z" }, + { url = "https://files.pythonhosted.org/packages/3b/32/9b8929bf384061ee1fe5d9c27c6f9776d3d824039ad4e14c88ec00c7808e/charset_normalizer-3.5.1-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:59171c6e45bf07d0d5cab3b0bf81d945035530f6873398b3b531c31184d46663", size = 381441, upload-time = "2026-08-15T08:19:26.038Z" }, + { url = "https://files.pythonhosted.org/packages/96/10/e9aa7923d3ddac652c99a1c5f7be494e737e151566a44abe018daf757f2c/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9dbdd9205662134957cf0c324f639bdc5031c0ca056e2369e238db75187c0f11", size = 241742, upload-time = "2026-08-15T08:19:27.532Z" }, + { url = "https://files.pythonhosted.org/packages/28/53/a2d249ebddf47b889a100c0bdcb61a2f9dbb8bc24ef325cc062e4f476877/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4b018dc5a0eee4676e38fe84a47a427816c590b93b55d9025274ec4d6ffc2dc", size = 235298, upload-time = "2026-08-15T08:19:29.274Z" }, + { url = "https://files.pythonhosted.org/packages/7d/07/469f78af590f7d5cd48e20d8dbfa3d66deeff9ba37768c04d886b5afd45c/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ced3fdd71aaa83ce593746c2edb42b7a59cb4c19c8b5c407781c72e493aae55a", size = 262500, upload-time = "2026-08-15T08:19:30.955Z" }, + { url = "https://files.pythonhosted.org/packages/55/66/3bb56a47f7dcba014055b1a1d33c6f08bbe9c1e74dba154cfa25f90ae885/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:19a3dd5aa73cef1c99687c4fc57db016a9c17104ae1185da88ba566a5d3bebe4", size = 258888, upload-time = "2026-08-15T08:19:32.458Z" }, + { url = "https://files.pythonhosted.org/packages/ff/c1/2adc2800903fb013210349313b710a5376856578d9e33e6b9a1d8b36714a/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cc5d36d96478aa9c60654bd932525bf32964c62a7281eafdf16d85003a8d6004", size = 250243, upload-time = "2026-08-15T08:19:33.94Z" }, + { url = "https://files.pythonhosted.org/packages/95/b5/a18d0dd1157ab655cc2cb14a545f4a4784bbad70ab3502412e36097502d9/charset_normalizer-3.5.1-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:04368edf83514385ffc3e1cfd4546e595f4f1272dd23ba437a93a9cc3741d47b", size = 249871, upload-time = "2026-08-15T08:19:35.413Z" }, + { url = "https://files.pythonhosted.org/packages/ad/c3/525f508cd1e58d0450ac55ed40ac75bc3a97482c59def5278456a5fbf03c/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:9b5db6052055d34d41230fb78d7c439c23dc536a9896f6cb039e8dd92cfc1263", size = 243580, upload-time = "2026-08-15T08:19:36.886Z" }, + { url = "https://files.pythonhosted.org/packages/7c/c1/49a91fe7e97c8140094ca5c64161ab623a70d9f636bf834eace14048acb5/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:252d099029bcbea642f2a06c4ed5046bdf8b5a8150b64afa5e027e88b106e5ee", size = 239807, upload-time = "2026-08-15T08:19:38.392Z" }, + { url = "https://files.pythonhosted.org/packages/d3/58/56a48c296601274c4689b864a8e2dfb209b81dfcb39472753ce95eea662b/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:6199d5606e2bbf2b096cf64d03f8b6790c91081d5ac866b8e7bb6422738cc60c", size = 264083, upload-time = "2026-08-15T08:19:39.856Z" }, + { url = "https://files.pythonhosted.org/packages/10/4c/dc48409274a1817ff349711d26c62aa0c597df865d4d69ef79160c859193/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:77efcff2b23071c349402ac1066667a3d011f62398d81408c9b88ad991747c9e", size = 250317, upload-time = "2026-08-15T08:19:41.53Z" }, + { url = "https://files.pythonhosted.org/packages/81/58/d325912115caec62d6bdd77bbab5e0b7da5d234a9f20affdffcbcb530d0b/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:a5cbd90ecf0fc62e64726917ad083b73001f0563657a87ec3c0b504e277dc90d", size = 258173, upload-time = "2026-08-15T08:19:43.07Z" }, + { url = "https://files.pythonhosted.org/packages/34/f7/b13b1ccae2c8ec63980d13be1890eb73f8aeabbfce02a24aabc0908788f5/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:4d26f14f041e83dd8edfd61f4cd4fa7285d31798b5bf1f28e70c367ba6c41d61", size = 251960, upload-time = "2026-08-15T08:19:44.587Z" }, + { url = "https://files.pythonhosted.org/packages/1e/25/ed3f9919c5aef8cc818be1f972f565f7610d7b2076b8ebb98839516ffc3c/charset_normalizer-3.5.1-cp315-cp315t-win32.whl", hash = "sha256:ac13b004224fb341e1e25a1ed5e19d32f57cdb2a403e01f003b46f051a550f6f", size = 191186, upload-time = "2026-08-15T08:19:46.293Z" }, + { url = "https://files.pythonhosted.org/packages/69/d5/43c2b3e9d8267092b913eb8b0603f0f71993c395632886bd37a7223f96cf/charset_normalizer-3.5.1-cp315-cp315t-win_amd64.whl", hash = "sha256:35aea775dc2bd5f54cd84a1cd2696cc3207c479cb9cf0bd346f0d343e4300ddb", size = 215947, upload-time = "2026-08-15T08:19:47.853Z" }, + { url = "https://files.pythonhosted.org/packages/a8/76/9aad3e9c8865e5e0efa9a7f6f81c37a67635a985145ecd44528a81e088ee/charset_normalizer-3.5.1-cp315-cp315t-win_arm64.whl", hash = "sha256:fb78f6e7fcd8ad785d28cd577168bc1aaee827b25bb8755638f694794ea98f0a", size = 193909, upload-time = "2026-08-15T08:19:49.383Z" }, + { url = "https://files.pythonhosted.org/packages/5b/97/fb4e82231aba271ffd775a1b4993b0defc4e3059f286ae41d9433409fe85/charset_normalizer-3.5.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:41876ee62a3dddf48ff1121ad8f0798032aa03f2fd35f21f34a4cab14f18d8d2", size = 331467, upload-time = "2026-08-15T08:19:50.959Z" }, + { url = "https://files.pythonhosted.org/packages/9f/2f/fe3f187327aac18e2d54e9d2b08e15d27bf9b642d9e51c219f130fc34d1a/charset_normalizer-3.5.1-cp37-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a6dac12ff6b846103483683f60c5f8fee205121adc58ffd87e90a90a3af69e99", size = 253057, upload-time = "2026-08-15T08:19:52.654Z" }, + { url = "https://files.pythonhosted.org/packages/d7/c7/9e48cee5c161fe24da823b61bf381921d77cb994a0a4de148e95018c1984/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cee5dd7c6fb5dd52a0fe2a740f9bc6e3593f5f8b1788bde49de02086f30182b2", size = 240930, upload-time = "2026-08-15T08:19:54.163Z" }, + { url = "https://files.pythonhosted.org/packages/49/e0/716601f3cc69be7b198951150c75ead1ece33c3c8036ff6ffa46029659a0/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:343fb4f2821043bd87095f7b08a1a181febc8e36ac64212143bbfd0a0e1bc235", size = 230822, upload-time = "2026-08-15T08:19:55.807Z" }, + { url = "https://files.pythonhosted.org/packages/d3/05/71bfc5caa0abcc45aea1f6a4d50ac68e59605ddc7666fe8494f4cd229665/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ae4a097991662cd4fff0ddc74e0fe7874f82e00042fa0ea00855645ed0c79598", size = 260037, upload-time = "2026-08-15T08:19:57.312Z" }, + { url = "https://files.pythonhosted.org/packages/c3/92/de7e32ed05341e7a9c4c877c318418197b7f2d66a3b68d561bf2ac57ca3e/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4b599739b93b2cbeded49645ae3c8d1405c29ddfbceac1545c87a3f9580a9e96", size = 255097, upload-time = "2026-08-15T08:19:59.056Z" }, + { url = "https://files.pythonhosted.org/packages/f5/7b/ade0a122600319dfa0b1000ab0f9731c94a817904cf3c5de408c73a4ede7/charset_normalizer-3.5.1-cp37-abi3-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b39b69b347e5e47a3b5b8cfc005c68c1ba347474e3960236c4944a8ecd174962", size = 250166, upload-time = "2026-08-15T08:20:00.612Z" }, + { url = "https://files.pythonhosted.org/packages/75/9c/019fbb9f4834491a160951349b1a3714439376f66e5f7cf18b4f18f0c7aa/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:a2028475ba855475b8b4d3cfeb4994269c967aea8b9892dfba907f4263a863a3", size = 241821, upload-time = "2026-08-15T08:20:02.321Z" }, + { url = "https://files.pythonhosted.org/packages/2b/b8/11d4840bfc99330cc7fbcc2681ee5a044553a6e77655508d8f9b2bff7b34/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:36047af20e17097c3bb9476c2b7655f2f7aa51322c0ba58c07695bedf755a950", size = 232529, upload-time = "2026-08-15T08:20:04.008Z" }, + { url = "https://files.pythonhosted.org/packages/18/96/2b3a21492d9f65171ac75d872f5018260013d00bfa0ff70ec9f179148cbd/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:4c4fb141a727957c93edfe5c32a26ceb6b5f6461d67146e2d39f51e16170bea8", size = 260348, upload-time = "2026-08-15T08:20:05.877Z" }, + { url = "https://files.pythonhosted.org/packages/d6/aa/a69a2028e8bd052476c245460ab19d7de595de084dd968f2d75cd50c3e25/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:2f293479cce755c75f1697e87c409b7ae4c555c7dfecb6e988ad13abba943031", size = 247234, upload-time = "2026-08-15T08:20:07.487Z" }, + { url = "https://files.pythonhosted.org/packages/35/8a/3d130aeabcaf3d2466af76b7b141c08d9e89c9016ab4b7cdd0f7dc2d1c62/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_s390x.whl", hash = "sha256:3588e376b3ea2eea84976f67273d679f229e24c66dce7b82ae45aef04ff6e072", size = 256917, upload-time = "2026-08-15T08:20:09.142Z" }, + { url = "https://files.pythonhosted.org/packages/80/c2/a7379b840292d0c1ab9fbd17d1f3967aa81794dc95bc74be8999d7fedcf7/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e199fb99720074809a7720f1c0b4d919eea8b87e88713e0f8f602f7bef543d9d", size = 254846, upload-time = "2026-08-15T08:20:10.727Z" }, + { url = "https://files.pythonhosted.org/packages/01/65/d43b714731bb2f40d4053dfa00ecfc1c5a301f8e3316c5db3a09af59fe94/charset_normalizer-3.5.1-cp37-abi3-win32.whl", hash = "sha256:dd732602a7009217f658d5863d12d79d373a4de0eebc111094bcdd3bb8e0a6cc", size = 174216, upload-time = "2026-08-15T08:20:12.334Z" }, + { url = "https://files.pythonhosted.org/packages/35/4f/b911ed898b26a09789eba9c9200c999aff6c61b4bafaf4838e56d1a1e1a3/charset_normalizer-3.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:70055ff39b97c99e7ae40ea3e393fb62aa2e44dbd9b29f8d14f42fb0025c3959", size = 199764, upload-time = "2026-08-15T08:20:13.908Z" }, + { url = "https://files.pythonhosted.org/packages/f0/a7/920baf467bfd9bf689f3b318340f37aee4572a71f162bd8db51da55ba4fa/charset_normalizer-3.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:87e4f41d375c0b9be2fb5251aee4b8a689169e134535aed81bf085c3b647451e", size = 287318, upload-time = "2026-08-15T08:20:15.551Z" }, + { url = "https://files.pythonhosted.org/packages/cc/61/d01fc49b8dea277640b55a9e15960dbca9fdc8c9fde18e572d39c59f4019/charset_normalizer-3.5.1-py3-none-any.whl", hash = "sha256:6df0ec430f9a831772c23ca5a224cba36517a58a84bb32c32bb59a9fa67c47f6", size = 68658, upload-time = "2026-08-15T08:20:43.306Z" }, +] + [[package]] name = "click" version = "8.4.2" @@ -361,6 +492,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/03/10388a42375ee7e4ac9b94eb2c5c569c8b5795e377e701c9ac3ad63de890/fastapi-0.141.1-py3-none-any.whl", hash = "sha256:bfb91aa2d334c61cb35ba9a116fc123b3d3df31640b801cf57a7a78ec3f603b3", size = 131954, upload-time = "2026-07-29T17:18:04.364Z" }, ] +[[package]] +name = "googleapis-common-protos" +version = "1.75.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/72/73/74bcab964c9a7a61f2bb71e8179b0f13e6fa98f7ce00fd168aab291e4a2e/googleapis_common_protos-1.75.1.tar.gz", hash = "sha256:d3042c6c5a2d4e67113104d6b6818b59b6bd92a197f2a91508e801fe815cf071", size = 150967, upload-time = "2026-08-06T06:24:51.972Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/51/186c02b8549b69ccda44429cf6ff5081e4b61a602ddfe6a8020d1be31d1b/googleapis_common_protos-1.75.1-py3-none-any.whl", hash = "sha256:28a1934bcd33b9c9da66ac301a0a4227e3367f095a17d0375cb98f0a09d93b79", size = 300626, upload-time = "2026-08-06T06:23:46.696Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -458,6 +601,9 @@ version = "2.12.6" source = { editable = "." } dependencies = [ { name = "certifi" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-sdk" }, { name = "pillow" }, { name = "rankweave" }, { name = "rdflib" }, @@ -489,6 +635,9 @@ requires-dist = [ { name = "fast-mlsirm", marker = "extra == 'backend'", git = "https://github.com/ContextualWisdomLab/fast-mlsirm.git?rev=5006c38286a4fa1d81bcf57eeed5ce27ae743f50" }, { name = "fastapi", marker = "extra == 'backend'", specifier = ">=0.115.0" }, { name = "httpx", marker = "extra == 'dev'", specifier = ">=0.27.0" }, + { name = "opentelemetry-api", specifier = ">=1.30.0" }, + { name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.30.0" }, + { name = "opentelemetry-sdk", specifier = ">=1.30.0" }, { name = "pillow", specifier = ">=12.3.0" }, { name = "psycopg2-binary", marker = "extra == 'dev'", specifier = ">=2.9.12" }, { name = "pyjwt", extras = ["crypto"], marker = "extra == 'backend'", specifier = ">=2.8.0" }, @@ -575,6 +724,87 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b4/07/458c344f0f0c178f4481dad5cca790626ffe4c34eabf9467069d06ee4999/numpy-2.5.2-cp315-cp315t-win_arm64.whl", hash = "sha256:5f8e00be2ec6f45f4e8a41a527f68d44a7d96fee92a650e4d8b1326f77f61e6e", size = 10748103, upload-time = "2026-08-09T13:48:24.21Z" }, ] +[[package]] +name = "opentelemetry-api" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ee/8b/aa9e2d8b8dfa7c946f7dec5d1f8f6ba8eca062f43509a06bdb5ce93d26c0/opentelemetry_api-1.44.0.tar.gz", hash = "sha256:67647e5e9566edcf421166fdf022b3537f818635daa852b289e34604dc6fb33a", size = 72406, upload-time = "2026-07-16T15:25:32.678Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/6f/a04e900f465ff3221ccc395522503e2d10e79fa21f2723c8e177aae1e0d1/opentelemetry_api-1.44.0-py3-none-any.whl", hash = "sha256:94b98c893a91b88657eaac1e3ba89618cdb85be6918196705354f34728b2cdef", size = 60018, upload-time = "2026-07-16T15:25:11.657Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-common" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-proto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/09/4d717852c1cf3f854b76c7110a5d00883bc3c99288b9b0dbcbeb9e306eb6/opentelemetry_exporter_otlp_proto_common-1.44.0.tar.gz", hash = "sha256:dc87a5a5bc58f149a56d1547e4691588fa12994cdc3bc039a694ccb3375862ac", size = 20202, upload-time = "2026-07-16T15:25:37.658Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/71/65fd9d54c10b860f87c045ccee1264cab7011268895d3528818a29c1172a/opentelemetry_exporter_otlp_proto_common-1.44.0-py3-none-any.whl", hash = "sha256:9a9fe61bba73d802904bc989f1d6b4a7b1ee40f06c40e98d6f85af65aaebb694", size = 17045, upload-time = "2026-07-16T15:25:18.201Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-http" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1a/87/95e2a5aaa795b4e2260d74e16df2d5541deb2ea9de010bcd615f4dee2654/opentelemetry_exporter_otlp_proto_http-1.44.0.tar.gz", hash = "sha256:c633d7270ad6b57cd4cfbe8b0007a9e2e7c0cb50bd6c50fe2a7b245f721a09d8", size = 25806, upload-time = "2026-07-16T15:25:39.162Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cd/d0/fdeb1a98d8d3a6205f5f297c51b4a9bfe65126ab60339669bbe3dd54c2e2/opentelemetry_exporter_otlp_proto_http-1.44.0-py3-none-any.whl", hash = "sha256:838592fce774c1c8bb7b9a0a7facbfa82e17be5a8a4e94cef10cb84ae026bae3", size = 21850, upload-time = "2026-07-16T15:25:20.006Z" }, +] + +[[package]] +name = "opentelemetry-proto" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/64/01/40ac4ae9a149263cc52c2cee200ddd80cb6d8db1a4610abf8eabce0fe771/opentelemetry_proto-1.44.0.tar.gz", hash = "sha256:c547a79c2f8c0c515d31509154682e5921c7cfd5ca67b70e1f9266e2c3e103f3", size = 46488, upload-time = "2026-07-16T15:25:45.34Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/7c/8be563d68e93bbefa5c8affb82ddcff91b3ad858ce49957ba7b16fd3e0ab/opentelemetry_proto-1.44.0-py3-none-any.whl", hash = "sha256:898b155a0e1557afd867478fb6158e8122a46329ca0bb8dc53cc55e98f017f56", size = 72483, upload-time = "2026-07-16T15:25:28.429Z" }, +] + +[[package]] +name = "opentelemetry-sdk" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5d/77/a6592cbc7c8d9bcc9d6757a9df45e04a7c585e3e6e7a13456da522b21109/opentelemetry_sdk-1.44.0.tar.gz", hash = "sha256:cebe7f65dc12f26ead75c6064de12fd2a9052e5060c0272d402cfa203aae123b", size = 208624, upload-time = "2026-07-16T15:25:46.078Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/23/ff077e61886ee020a17ce9c8b6fa11c601c8d8345b09ea24f605445df62a/opentelemetry_sdk-1.44.0-py3-none-any.whl", hash = "sha256:df081c4c6bcfdb1211e3e86140376792643128a25f8d72d1d27675936e7e96ad", size = 137221, upload-time = "2026-07-16T15:25:29.534Z" }, +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.65b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8f/73/0cbdebcb4cf545fdd328da14f5137e37d0770c3f26185e478b0d15d94f50/opentelemetry_semantic_conventions-0.65b0.tar.gz", hash = "sha256:f9b2b81e9d5b64f11bc952075e7e9c7fb0aab075c7fd1c46d597f1b919852d60", size = 148774, upload-time = "2026-07-16T15:25:46.902Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a6/0e/49df70d9b81fb5cbae4bbf2a49d865b09bcbcbc4eb53f5851b1027738d78/opentelemetry_semantic_conventions-0.65b0-py3-none-any.whl", hash = "sha256:1cacde7b0ad306f84c5ef08c3dbe1bbaf20165bba6f8bff43b670e555a086bcb", size = 204645, upload-time = "2026-07-16T15:25:30.688Z" }, +] + [[package]] name = "packaging" version = "26.3" @@ -664,6 +894,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "protobuf" +version = "7.36.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a7/e7/0553e21d25ca4d9f573135775348a372c3ec34a93a71d5f297c3bac38341/protobuf-7.36.0.tar.gz", hash = "sha256:e8e09cb0d794c6687926fa558a8a6e72aa10edb997d5ca61da0765f12a3e00ea", size = 510034, upload-time = "2026-08-20T16:34:01.071Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/ae/58e3ca96cb2e118cc546b677359b3c6659f79a140935c08dec94c7998585/protobuf-7.36.0-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:9103532dffd80c6fab7e50c65a31007680a06eb57537d437bb1b35812c138a37", size = 453256, upload-time = "2026-08-20T16:33:53.945Z" }, + { url = "https://files.pythonhosted.org/packages/f0/15/5162230af4912697f0fe406f6800f80760945babcff0e2c2fe6c84ef2d5d/protobuf-7.36.0-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:bf94a5917c71058262de683669bc0a797a7669d3de71f0b36d058e3194f47b44", size = 341436, upload-time = "2026-08-20T16:33:55.134Z" }, + { url = "https://files.pythonhosted.org/packages/d7/09/1670b2bfc9a45e807e520c3e9be36524db9ccc7dc05ea17af7681cabdc61/protobuf-7.36.0-cp310-abi3-manylinux2014_s390x.whl", hash = "sha256:3297e60abdff301e5f74393d87f6cc59dacab5f024a89548a6e8de1d26576b16", size = 354440, upload-time = "2026-08-20T16:33:56.077Z" }, + { url = "https://files.pythonhosted.org/packages/c7/f8/bd5804695ba400e423c33fd4d9f58c28d86633d5ba1945c36ff3967d98cb/protobuf-7.36.0-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:70f5ec8eb0da81a44360c0dc0beac99a0d78071d21956a7076bae8bd2051841b", size = 340439, upload-time = "2026-08-20T16:33:56.992Z" }, + { url = "https://files.pythonhosted.org/packages/ef/9f/acd02338235a3e7d03168c4303478347b7624fc8189ff4e7f0d2654bbe86/protobuf-7.36.0-cp310-abi3-win32.whl", hash = "sha256:7326fd717bdc419162a735938d89d4032332bcc3408804012b24ff3a37086071", size = 440216, upload-time = "2026-08-20T16:33:57.99Z" }, + { url = "https://files.pythonhosted.org/packages/0e/4e/12cb93270967a2affff5b3f720694700d4d87712a67afd05c8cb3f6fa52c/protobuf-7.36.0-cp310-abi3-win_amd64.whl", hash = "sha256:1781cc1de61249b750848029bca452c0a8b7e990080316b9bbc2518b2117b488", size = 453731, upload-time = "2026-08-20T16:33:58.951Z" }, + { url = "https://files.pythonhosted.org/packages/01/c3/629999e78d46c1115c11886d51c6bd68c17ce4a944f1ea3e153a91316a33/protobuf-7.36.0-py3-none-any.whl", hash = "sha256:53374d53fc29a67f7dbbf0ade47d7526a0f0137bf0f9c90e48d8a60790ef748c", size = 177024, upload-time = "2026-08-20T16:34:00.053Z" }, +] + [[package]] name = "psycopg2-binary" version = "2.9.12" @@ -933,6 +1178,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/66/9d/c5731f6e3608663d4d3656fd8d3aecee8b509c3082818f5a13eae925baea/redis-8.1.0-py3-none-any.whl", hash = "sha256:a4fe1aac3d3b3cc791d4b3d5931c5a956045dc951ee74d1c913ee3ac4d2ee9fb", size = 560618, upload-time = "2026-07-30T08:50:58.497Z" }, ] +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + [[package]] name = "starlette" version = "1.6.0" @@ -976,6 +1236,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147", size = 14750, upload-time = "2026-08-12T12:37:24.648Z" }, ] +[[package]] +name = "urllib3" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, +] + [[package]] name = "uvicorn" version = "0.52.1" From 707dcfce18dbe24799d0395385239d342604365f Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 22:18:47 +0900 Subject: [PATCH 02/29] fix: classify server failures in OTel telemetry --- CHANGELOG.d/2.13.2-otel-server-diagnostics.md | 4 + backend/app/main.py | 57 +++++-- backend/tests/test_server_diagnostics.py | 118 ++++++++++++++ docs/adr/0122-otel-session-observability.md | 29 ++-- docs/doctoring/OPENTELEMETRY_REFERENCES.md | 4 +- lineageweave/observability.py | 153 +++++++++++++++++- 6 files changed, 338 insertions(+), 27 deletions(-) create mode 100644 CHANGELOG.d/2.13.2-otel-server-diagnostics.md create mode 100644 backend/tests/test_server_diagnostics.py diff --git a/CHANGELOG.d/2.13.2-otel-server-diagnostics.md b/CHANGELOG.d/2.13.2-otel-server-diagnostics.md new file mode 100644 index 000000000..2d556d6ef --- /dev/null +++ b/CHANGELOG.d/2.13.2-otel-server-diagnostics.md @@ -0,0 +1,4 @@ +## 2.13.2 + +- Add buyer-safe Global Ask and post-chat failures with bounded OpenTelemetry + metrics, traces, and structured server diagnostics for GRC consumption. diff --git a/backend/app/main.py b/backend/app/main.py index 46dea0e09..fff829a48 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -183,10 +183,13 @@ from backend.app.demo_scope import ( fetch_demo_corporate_entity_ids, has_real_source_context, - is_demo_scope, ) from lineageweave.http_client import HttpClientError -from lineageweave.observability import configure_telemetry +from lineageweave.observability import ( + configure_telemetry, + record_server_failure, + traced, +) _POST_READ = "post_read" _POST_ADMIN = "post_admin" @@ -2580,20 +2583,36 @@ async def chat_about_post( with use_llm_metadata(post_metadata): client = _post_chat_client() if not client.available: + record_server_failure( + "post_chat", + RuntimeError("orchestrator unavailable"), + outcome="provider_unavailable", + ) raise HTTPException( status.HTTP_503_SERVICE_UNAVAILABLE, - "Post chat is unavailable: set ORCHESTRATOR_BASE_URL / ORCHESTRATOR_API_KEY", + "Post chat is temporarily unavailable. " + "Saved evidence is still available.", ) sources = await gather_chat_sources( conn, post_id, lambda row: _can_see_post(account, row), vision_client=_vision_client() ) try: - with use_llm_metadata(post_metadata): - answer = await asyncio.to_thread(client.answer, question, sources) - except (HttpClientError, KeyError, OSError, ValueError) as exc: + with traced("lineageweave.api.post_chat", {"operation_code": "post_chat"}): + with use_llm_metadata(post_metadata): + answer = await asyncio.to_thread(client.answer, question, sources) + except (HttpClientError, KeyError, OSError, TypeError, ValueError) as exc: + record_server_failure("post_chat", exc, outcome="provider_unavailable") + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + "Post chat is temporarily unavailable. " + "Saved evidence is still available.", + ) from exc + except Exception as exc: + record_server_failure("post_chat", exc, outcome="internal_error") raise HTTPException( status.HTTP_503_SERVICE_UNAVAILABLE, - "Post chat is unavailable: contextual-orchestrator returned no complete evidence object", + "Post chat is temporarily unavailable. " + "Saved evidence is still available.", ) from exc cited_ids = list(answer.cited_post_ids) async with pool.acquire() as conn: @@ -2627,9 +2646,15 @@ async def ask_agent( _require_post_read(account) client = _post_chat_client() if not client.available: + record_server_failure( + "global_ask", + RuntimeError("orchestrator unavailable"), + outcome="provider_unavailable", + ) raise HTTPException( status.HTTP_503_SERVICE_UNAVAILABLE, - "Ask Agent is unavailable: set ORCHESTRATOR_BASE_URL / ORCHESTRATOR_API_KEY", + "Ask Agent is temporarily unavailable. " + "Saved evidence is still available.", ) async with pool.acquire() as conn: sources = await gather_global_chat_sources( @@ -2648,11 +2673,21 @@ async def ask_agent( "next_action": "No authorized source posts are available for this question.", } try: - answer = await asyncio.to_thread(client.answer, question, sources) - except (HttpClientError, KeyError, OSError, ValueError) as exc: + with traced("lineageweave.api.global_ask", {"operation_code": "global_ask"}): + answer = await asyncio.to_thread(client.answer, question, sources) + except (HttpClientError, KeyError, OSError, TypeError, ValueError) as exc: + record_server_failure("global_ask", exc, outcome="provider_unavailable") + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + "Ask Agent is temporarily unavailable. " + "Saved evidence is still available.", + ) from exc + except Exception as exc: + record_server_failure("global_ask", exc, outcome="internal_error") raise HTTPException( status.HTTP_503_SERVICE_UNAVAILABLE, - f"Ask Agent is unavailable: {exc}", + "Ask Agent is temporarily unavailable. " + "Saved evidence is still available.", ) from exc cited_ids = list(answer.cited_post_ids) return { diff --git a/backend/tests/test_server_diagnostics.py b/backend/tests/test_server_diagnostics.py new file mode 100644 index 000000000..b82c260ab --- /dev/null +++ b/backend/tests/test_server_diagnostics.py @@ -0,0 +1,118 @@ +"""Buyer-safe API failures and bounded OpenTelemetry diagnostics.""" + +from __future__ import annotations + +import asyncio +import logging +from types import SimpleNamespace +from typing import Self + +import pytest + +from backend.app import main +from lineageweave import observability + + +class _Pool: + """Minimal async pool boundary for the endpoint unit tests.""" + + def acquire(self) -> Self: + return self + + async def __aenter__(self) -> Self: + return self + + async def __aexit__(self, *exc_info: object) -> None: + return None + + +class _FailingClient: + """Orchestrator-shaped client that raises one controlled exception.""" + + available = True + + def __init__(self, exc: BaseException) -> None: + self._exc = exc + + def answer(self, question: str, sources: object) -> object: + raise self._exc + + +def _call_ask(monkeypatch: pytest.MonkeyPatch, exc: BaseException) -> None: + async def _sources(*args: object, **kwargs: object) -> list[object]: + return [SimpleNamespace(post_id="synthetic-post-1")] + + account = SimpleNamespace( + corporate_entity_ids=(), + has_permission=lambda permission: permission == "post_read", + ) + monkeypatch.setattr(main, "gather_global_chat_sources", _sources) + monkeypatch.setattr(main, "_post_chat_client", lambda: _FailingClient(exc)) + with pytest.raises(main.HTTPException) as raised: + asyncio.run( + main.ask_agent( + main.GlobalAskRequest(question="synthetic question"), + account=account, + pool=_Pool(), + ) + ) + assert raised.value.status_code == 503 + assert raised.value.detail == ( + "Ask Agent is temporarily unavailable. Saved evidence is still available." + ) + + +def test_global_ask_provider_failure_is_buyer_safe_and_classified( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """Known provider/schema errors produce a provider-unavailable signal.""" + counter_calls: list[tuple[int, dict[str, str]]] = [] + + class _Counter: + def add(self, value: int, attributes: dict[str, str]) -> None: + counter_calls.append((value, attributes)) + + monkeypatch.setattr(observability, "_FAILURE_COUNTER", _Counter()) + caplog.set_level(logging.WARNING, logger="lineageweave.observability") + sensitive = "provider response body must not escape" + _call_ask(monkeypatch, ValueError(sensitive)) + + record = next( + item for item in caplog.records if item.msg == "lineageweave.server_failure" + ) + assert record.operation_code == "global_ask" + assert record.failure_outcome == "provider_unavailable" + assert record.error_type == "ValueError" + assert record.stack_trace == "" + assert sensitive not in caplog.text + assert counter_calls == [ + ( + 1, + { + "lineageweave.operation_code": "global_ask", + "lineageweave.failure_outcome": "provider_unavailable", + }, + ) + ] + + +def test_global_ask_internal_failure_is_buyer_safe_and_keeps_stack_without_value( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """Unexpected defects are traceable without exposing their exception value.""" + caplog.set_level(logging.WARNING, logger="lineageweave.observability") + sensitive = "internal prompt-like value must not escape" + try: + raise AttributeError(sensitive) + except AttributeError as exc: + _call_ask(monkeypatch, exc) + + record = next( + item for item in caplog.records if item.msg == "lineageweave.server_failure" + ) + assert record.operation_code == "global_ask" + assert record.failure_outcome == "internal_error" + assert record.error_type == "AttributeError" + assert record.stack_trace + assert sensitive not in record.stack_trace + assert sensitive not in caplog.text diff --git a/docs/adr/0122-otel-session-observability.md b/docs/adr/0122-otel-session-observability.md index cbd685e12..3811ced63 100644 --- a/docs/adr/0122-otel-session-observability.md +++ b/docs/adr/0122-otel-session-observability.md @@ -28,21 +28,32 @@ control contract in [ADR 0009](https://github.com/ContextualWisdomLab/governance 3. LineageWeave emits bounded HTTP and Valkey operation spans. Valkey spans identify the operation and logical stream kind, not the stream key, post body, summary, actor, source identifiers, token, or provider response. -4. Failure logs contain operation, error type, status, and the bounded session - correlation only. They do not become a second evidence database. GRC may - consume aggregate control evidence and OTLP-derived SLO signals through its - existing contracts; LineageWeave does not copy GRC tables or credentials. -5. No ad hoc session table is introduced. The existing normalized post-scoped +4. Failure telemetry uses two fixed outcomes: `provider_unavailable` for + explicitly classified provider, transport, or schema failures, and + `internal_error` for unexpected programming failures. The counter labels + contain only operation code and outcome, so high-cardinality session IDs and + exception classes remain in bounded structured logs and traces instead of + metric labels. +5. Failure logs contain operation, error type, status, and the bounded session + correlation only. Unexpected failures may include a bounded stack trace, + but never the exception value, prompt, response, source body, credential, + actor, or tenant identifier. They do not become a second evidence database. + GRC may consume aggregate control evidence and OTLP-derived SLO signals + through its existing contracts; LineageWeave does not copy GRC tables or + credentials. +6. No ad hoc session table is introduced. The existing normalized post-scoped session metadata remains the source of correlation. ## Consequences Operators can follow a slow or failed post-content job from the LineageWeave HTTP client through contextual-orchestrator and Valkey without exposing source -content. An OTLP collector is a deployment concern, not a local default, so a -developer stack remains usable without a telemetry backend. Raw session IDs -remain correlation data and must not be used as tenant, actor, or evidence -labels in GRC dashboards. +content. Global Ask and post chat return a buyer-safe generic 503 while GRC +can distinguish provider unavailability from an internal defect. An OTLP +collector is a deployment concern, not a local default, so a developer stack +remains usable without a telemetry backend. Raw session IDs remain +correlation data and must not be used as tenant, actor, or evidence labels in +GRC dashboards. ## References diff --git a/docs/doctoring/OPENTELEMETRY_REFERENCES.md b/docs/doctoring/OPENTELEMETRY_REFERENCES.md index 4cec4d416..9efb02821 100644 --- a/docs/doctoring/OPENTELEMETRY_REFERENCES.md +++ b/docs/doctoring/OPENTELEMETRY_REFERENCES.md @@ -20,7 +20,9 @@ | Post correlation | Existing ADR 0071 session metadata plus `X-LineageWeave-Session-Id` | Correlation only; not identity or authorization | | LLM/VISION/embedding transport | `lineageweave.http_client.post_json` | Method, peer, bounded path, status; no body or credential | | Valkey queue | `backend/app/*worker.py` and stream producers | Operation and logical stream kind; no stream key or event content | -| Export | OTEL_EXPORTER_OTLP_ENDPOINT | Disabled by default; base URL normalized to /v1/traces | +| Server failure metric | `lineageweave.server.failures` | Fixed operation/outcome labels; no session or exception labels | +| Server failure log/trace | `record_server_failure` | Error class and bounded stack only; no exception value or source content | +| Export | OTEL_EXPORTER_OTLP_ENDPOINT | Disabled by default; base URL normalized to /v1/traces and /v1/metrics | The GRC repository remains the organization control and evidence owner. This repository emits operational signals and does not copy GRC tables or persist diff --git a/lineageweave/observability.py b/lineageweave/observability.py index e7734f4cf..6f74d60b2 100644 --- a/lineageweave/observability.py +++ b/lineageweave/observability.py @@ -9,15 +9,17 @@ import logging import os +import traceback from collections.abc import Iterator, Mapping from contextlib import contextmanager from typing import Any try: - from opentelemetry import trace + from opentelemetry import metrics, trace from opentelemetry.propagate import inject as _otel_inject from opentelemetry.trace import Status, StatusCode except ImportError: # pragma: no cover - dependency is declared by the project + metrics = None # type: ignore[assignment] trace = None # type: ignore[assignment] _otel_inject = None Status = None # type: ignore[assignment,misc] @@ -26,14 +28,27 @@ _LOGGER = logging.getLogger(__name__) _CONFIGURED = False _TRACER_NAME = "lineageweave" +_FAILURE_COUNTER: Any = None +_SERVER_FAILURE_OUTCOMES = {"provider_unavailable", "internal_error"} def _otlp_trace_endpoint(endpoint: str) -> str: """Turn an OTLP base endpoint into the explicit HTTP traces endpoint.""" + return _otlp_signal_endpoint(endpoint, "traces") + + +def _otlp_signal_endpoint(endpoint: str, signal: str) -> str: + """Turn an OTLP base endpoint into one explicit HTTP signal endpoint.""" normalized = endpoint.rstrip("/") - if normalized.casefold().endswith("/v1/traces"): + suffix = f"/v1/{signal}" + if normalized.casefold().endswith(suffix): return normalized - return f"{normalized}/v1/traces" + return f"{normalized}{suffix}" + + +def _otlp_metric_endpoint(endpoint: str) -> str: + """Turn an OTLP base endpoint into the explicit HTTP metrics endpoint.""" + return _otlp_signal_endpoint(endpoint, "metrics") def current_session_id() -> str | None: @@ -74,7 +89,7 @@ def _safe_attributes( def configure_telemetry(service_name: str = "lineageweave") -> None: - """Configure one OTLP trace provider when an operator supplied an endpoint.""" + """Configure OTLP traces and bounded failure metrics when enabled.""" global _CONFIGURED if _CONFIGURED or os.getenv("OTEL_SDK_DISABLED", "").lower() == "true": return @@ -90,7 +105,7 @@ def configure_telemetry(service_name: str = "lineageweave") -> None: from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import BatchSpanProcessor except ImportError: # pragma: no cover - guarded by the runtime extra - _LOGGER.warning("OpenTelemetry SDK/exporter is unavailable") + _LOGGER.warning("OpenTelemetry trace SDK/exporter is unavailable") return resource = Resource.create({ @@ -104,6 +119,129 @@ def configure_telemetry(service_name: str = "lineageweave") -> None: ) ) trace.set_tracer_provider(provider) + if metrics is None: + return + try: + from opentelemetry.exporter.otlp.proto.http.metric_exporter import ( + OTLPMetricExporter, + ) + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader + except ImportError: # pragma: no cover - guarded by the runtime extra + _LOGGER.warning("OpenTelemetry metric SDK/exporter is unavailable") + return + metrics.set_meter_provider( + MeterProvider( + resource=resource, + metric_readers=[ + PeriodicExportingMetricReader( + OTLPMetricExporter(endpoint=_otlp_metric_endpoint(endpoint)) + ) + ], + ) + ) + + +def _failure_counter() -> Any: + """Return the OTel counter without making telemetry a request dependency.""" + global _FAILURE_COUNTER + if _FAILURE_COUNTER is None and metrics is not None: + _FAILURE_COUNTER = metrics.get_meter(_TRACER_NAME).create_counter( + "lineageweave.server.failures", + description="Server failures classified by operation and outcome", + ) + return _FAILURE_COUNTER + + +def _stack_trace_without_exception(exc: BaseException) -> str: + """Bound a stack trace while excluding the exception value/message.""" + if exc.__traceback__ is None: + return "" + return "".join(traceback.format_tb(exc.__traceback__))[:4096] + + +def _annotate_failure_span( + span: Any, + operation_code: str, + outcome: str, + error_type: str, + stack_trace: str, +) -> None: + """Attach only bounded classification data to an active span.""" + safe = _safe_attributes( + { + "lineageweave.operation_code": operation_code, + "lineageweave.failure_outcome": outcome, + "lineageweave.error_type": error_type, + } + ) + for key, value in safe.items(): + span.set_attribute(key, value) + event_attributes: dict[str, str] = {"exception.type": error_type} + if stack_trace: + event_attributes["exception.stacktrace"] = stack_trace + span.add_event("exception", event_attributes) + if Status is not None and StatusCode is not None: + span.set_status(Status(StatusCode.ERROR)) + + +def record_server_failure( + operation_code: str, + exc: BaseException, + *, + outcome: str, +) -> None: + """Record a classified server failure without storing exception content. + + ``provider_unavailable`` and ``internal_error`` are the only metric + outcomes. The error class is retained in logs and spans, while the + exception value is intentionally never serialized. + """ + if outcome not in _SERVER_FAILURE_OUTCOMES: + raise ValueError(f"unsupported server failure outcome: {outcome}") + bounded_operation = operation_code.strip()[:64] + error_type = type(exc).__name__[:128] + session_id = current_session_id() or "" + counter = _failure_counter() + if counter is not None: + try: + counter.add( + 1, + { + "lineageweave.operation_code": bounded_operation, + "lineageweave.failure_outcome": outcome, + }, + ) + except Exception: # noqa: BLE001 # telemetry failure must not mask API failure + _LOGGER.warning("telemetry.metric_recording_failed") + + stack_trace = ( + _stack_trace_without_exception(exc) if outcome == "internal_error" else "" + ) + if trace is not None: + current = trace.get_current_span() + if current is not None and current.is_recording(): + _annotate_failure_span( + current, bounded_operation, outcome, error_type, stack_trace + ) + else: + tracer = trace.get_tracer(_TRACER_NAME) + with tracer.start_as_current_span("lineageweave.server.failure") as span: + _annotate_failure_span( + span, bounded_operation, outcome, error_type, stack_trace + ) + + _LOGGER.log( + logging.ERROR if outcome == "internal_error" else logging.WARNING, + "lineageweave.server_failure", + extra={ + "operation_code": bounded_operation, + "failure_outcome": outcome, + "error_type": error_type, + "session_id": session_id, + "stack_trace": stack_trace, + }, + ) @contextmanager @@ -124,7 +262,10 @@ def traced( yield span except Exception as exc: if Status is not None and StatusCode is not None: - span.record_exception(exc) + span.add_event( + "exception", + {"exception.type": type(exc).__name__[:128]}, + ) span.set_status(Status(StatusCode.ERROR)) _LOGGER.warning( "telemetry.operation_failed operation=%s error_type=%s session_id=%s", From 005f066c0ad292b1406cdb84fd247daae51ae26c Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 22:23:39 +0900 Subject: [PATCH 03/29] fix: redact worker failure details --- backend/app/post_content_worker.py | 7 ++++--- backend/tests/test_server_diagnostics.py | 10 ++++++++++ lineageweave/observability.py | 2 +- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/backend/app/post_content_worker.py b/backend/app/post_content_worker.py index 2077d90f7..37aaa9184 100644 --- a/backend/app/post_content_worker.py +++ b/backend/app/post_content_worker.py @@ -38,6 +38,7 @@ _RECOVERY_INTERVAL_SECONDS = 30.0 _INCOMPLETE_FAILURE_CODE = "post_content_ingestion_incomplete" _ATTEMPT_LIMIT_FAILURE_CODE = "post_content_ingestion_attempt_limit" +_UNEXPECTED_FAILURE_DETAIL = "post-content ingestion failed; inspect server telemetry" async def _stream_tail(client: redis.Redis) -> str: @@ -270,13 +271,13 @@ async def process_post_content_job( expected_attempt_count=attempt_count, ) return - except Exception as exc: # noqa: BLE001 - durable failure is recorded for retry. - _logger.exception("post content ingestion failed for post_id=%s", post_id) + except Exception: # noqa: BLE001 - durable failure is recorded for retry. + _logger.exception("post content ingestion failed") await _finish_failed_job( pool, post_id, failure_code="post_content_ingestion_failed", - detail_text=str(exc)[:1000], + detail_text=_UNEXPECTED_FAILURE_DETAIL, expected_attempt_count=attempt_count, ) return diff --git a/backend/tests/test_server_diagnostics.py b/backend/tests/test_server_diagnostics.py index b82c260ab..32edecc81 100644 --- a/backend/tests/test_server_diagnostics.py +++ b/backend/tests/test_server_diagnostics.py @@ -116,3 +116,13 @@ def test_global_ask_internal_failure_is_buyer_safe_and_keeps_stack_without_value assert record.stack_trace assert sensitive not in record.stack_trace assert sensitive not in caplog.text + + +def test_telemetry_without_endpoint_does_not_latch_configuration( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A disabled first call still permits a later operator configuration.""" + monkeypatch.setattr(observability, "_CONFIGURED", False) + monkeypatch.delenv("OTEL_EXPORTER_OTLP_ENDPOINT", raising=False) + observability.configure_telemetry() + assert observability._CONFIGURED is False diff --git a/lineageweave/observability.py b/lineageweave/observability.py index 6f74d60b2..0891fc9da 100644 --- a/lineageweave/observability.py +++ b/lineageweave/observability.py @@ -93,7 +93,6 @@ def configure_telemetry(service_name: str = "lineageweave") -> None: global _CONFIGURED if _CONFIGURED or os.getenv("OTEL_SDK_DISABLED", "").lower() == "true": return - _CONFIGURED = True endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "").strip() if trace is None or not endpoint: return @@ -119,6 +118,7 @@ def configure_telemetry(service_name: str = "lineageweave") -> None: ) ) trace.set_tracer_provider(provider) + _CONFIGURED = True if metrics is None: return try: From c9e95e738062936c8d6239989860fc4fd6fabbf3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 22:33:23 +0900 Subject: [PATCH 04/29] fix: flush telemetry and label TEPP transport --- backend/app/analysis_run_start.py | 8 +++++- backend/app/main.py | 8 ++++-- lineageweave/http_client.py | 5 +++- lineageweave/observability.py | 44 ++++++++++++++++++++++++------- tests/test_observability.py | 26 +++++++++++++++++- tests/test_tepp_client.py | 18 +++++++++++-- 6 files changed, 92 insertions(+), 17 deletions(-) diff --git a/backend/app/analysis_run_start.py b/backend/app/analysis_run_start.py index 2387d940b..a06cebc11 100644 --- a/backend/app/analysis_run_start.py +++ b/backend/app/analysis_run_start.py @@ -102,7 +102,13 @@ def configured_tepp_client(transport_url: str = "", api_key: str = "") -> TeppCl def transport(payload: dict[str, Any]) -> dict[str, Any]: try: headers = {"authorization": f"Bearer {api_key}"} if api_key.strip() else {} - return post_json(url, payload, headers=headers, timeout=30.0) + return post_json( + url, + payload, + headers=headers, + timeout=30.0, + service_peer_name="tepp", + ) except (HttpClientError, OSError, ValueError, TypeError) as exc: raise TeppNotAvailable(str(exc)) from exc diff --git a/backend/app/main.py b/backend/app/main.py index fff829a48..b6f983258 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -188,6 +188,7 @@ from lineageweave.observability import ( configure_telemetry, record_server_failure, + shutdown_telemetry, traced, ) @@ -233,8 +234,11 @@ async def lifespan(app: FastAPI): app.state.post_content_worker, return_exceptions=True, ) - await app.state.pool.close() - await app.state.valkey.aclose() + try: + await app.state.pool.close() + await app.state.valkey.aclose() + finally: + shutdown_telemetry() app = FastAPI(title="LineageWeave API", lifespan=lifespan) diff --git a/lineageweave/http_client.py b/lineageweave/http_client.py index 3d6813334..e1428b726 100644 --- a/lineageweave/http_client.py +++ b/lineageweave/http_client.py @@ -106,12 +106,15 @@ def post_json( *, headers: dict[str, str], timeout: float, + service_peer_name: str = "contextual-orchestrator", ) -> dict: """POST ``payload`` as JSON to ``url`` and return the decoded object. Raises: ValueError: ``url`` is not an ``http`` / ``https`` URL with a host. HttpClientError: the server responded with HTTP >= 400 or non-JSON. + + ``service_peer_name`` is a bounded service name used for the request span. """ request_payload = payload request_metadata = current_llm_metadata() @@ -136,7 +139,7 @@ def post_json( "http.request.method": "POST", "server.address": hostname, "url.path": parsed.path or "/", - "service.peer.name": "contextual-orchestrator", + "service.peer.name": service_peer_name, }, ) as span: inject_trace_context(request_headers) diff --git a/lineageweave/observability.py b/lineageweave/observability.py index 0891fc9da..c8f41dc32 100644 --- a/lineageweave/observability.py +++ b/lineageweave/observability.py @@ -29,6 +29,8 @@ _CONFIGURED = False _TRACER_NAME = "lineageweave" _FAILURE_COUNTER: Any = None +_TRACE_PROVIDER: Any = None +_METER_PROVIDER: Any = None _SERVER_FAILURE_OUTCOMES = {"provider_unavailable", "internal_error"} @@ -90,7 +92,7 @@ def _safe_attributes( def configure_telemetry(service_name: str = "lineageweave") -> None: """Configure OTLP traces and bounded failure metrics when enabled.""" - global _CONFIGURED + global _CONFIGURED, _TRACE_PROVIDER, _METER_PROVIDER if _CONFIGURED or os.getenv("OTEL_SDK_DISABLED", "").lower() == "true": return endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "").strip() @@ -118,6 +120,7 @@ def configure_telemetry(service_name: str = "lineageweave") -> None: ) ) trace.set_tracer_provider(provider) + _TRACE_PROVIDER = provider _CONFIGURED = True if metrics is None: return @@ -130,16 +133,37 @@ def configure_telemetry(service_name: str = "lineageweave") -> None: except ImportError: # pragma: no cover - guarded by the runtime extra _LOGGER.warning("OpenTelemetry metric SDK/exporter is unavailable") return - metrics.set_meter_provider( - MeterProvider( - resource=resource, - metric_readers=[ - PeriodicExportingMetricReader( - OTLPMetricExporter(endpoint=_otlp_metric_endpoint(endpoint)) - ) - ], - ) + meter_provider = MeterProvider( + resource=resource, + metric_readers=[ + PeriodicExportingMetricReader( + OTLPMetricExporter(endpoint=_otlp_metric_endpoint(endpoint)) + ) + ], ) + metrics.set_meter_provider(meter_provider) + _METER_PROVIDER = meter_provider + + +def shutdown_telemetry() -> None: + """Flush configured OTLP providers without masking application shutdown.""" + global _TRACE_PROVIDER, _METER_PROVIDER, _FAILURE_COUNTER + for provider_name, provider in ( + ("trace", _TRACE_PROVIDER), + ("metric", _METER_PROVIDER), + ): + if provider is None: + continue + try: + provider.shutdown() + except Exception: # noqa: BLE001 - telemetry must not mask shutdown + _LOGGER.warning( + "telemetry.provider_shutdown_failed provider=%s", + provider_name, + ) + _TRACE_PROVIDER = None + _METER_PROVIDER = None + _FAILURE_COUNTER = None def _failure_counter() -> Any: diff --git a/tests/test_observability.py b/tests/test_observability.py index e67c2fc38..4ae04509b 100644 --- a/tests/test_observability.py +++ b/tests/test_observability.py @@ -1,10 +1,11 @@ """Tests for prompt-safe session propagation and tracing boundaries.""" -from lineageweave import http_client +from lineageweave import http_client, observability from lineageweave.llm_context import use_llm_metadata from lineageweave.observability import ( _otlp_trace_endpoint, current_session_id, + shutdown_telemetry, traced, ) @@ -56,3 +57,26 @@ def test_otlp_base_endpoint_gets_trace_signal_path(): """A configured collector base URL receives the HTTP traces signal path.""" assert _otlp_trace_endpoint("http://collector:4318") == "http://collector:4318/v1/traces" assert _otlp_trace_endpoint("http://collector:4318/v1/traces/") == "http://collector:4318/v1/traces" + + +def test_shutdown_telemetry_flushes_configured_providers(monkeypatch): + """Application shutdown flushes traces and metrics without a raw error.""" + calls = [] + + class _Provider: + def __init__(self, name): + self.name = name + + def shutdown(self): + calls.append(self.name) + + monkeypatch.setattr(observability, "_TRACE_PROVIDER", _Provider("trace")) + monkeypatch.setattr(observability, "_METER_PROVIDER", _Provider("metric")) + monkeypatch.setattr(observability, "_FAILURE_COUNTER", object()) + + shutdown_telemetry() + + assert calls == ["trace", "metric"] + assert observability._TRACE_PROVIDER is None + assert observability._METER_PROVIDER is None + assert observability._FAILURE_COUNTER is None diff --git a/tests/test_tepp_client.py b/tests/test_tepp_client.py index ea87d5558..10eb33fd5 100644 --- a/tests/test_tepp_client.py +++ b/tests/test_tepp_client.py @@ -55,8 +55,21 @@ def fake_transport(payload: dict) -> dict: def test_configured_transport_sends_optional_bearer_key(monkeypatch: pytest.MonkeyPatch) -> None: received = {} - def fake_post_json(url: str, payload: dict, *, headers: dict, timeout: float) -> dict: - received.update(url=url, payload=payload, headers=headers, timeout=timeout) + def fake_post_json( + url: str, + payload: dict, + *, + headers: dict, + timeout: float, + service_peer_name: str, + ) -> dict: + received.update( + url=url, + payload=payload, + headers=headers, + timeout=timeout, + service_peer_name=service_peer_name, + ) return {"status": "accepted"} monkeypatch.setattr("backend.app.analysis_run_start.post_json", fake_post_json) @@ -66,3 +79,4 @@ def fake_post_json(url: str, payload: dict, *, headers: dict, timeout: float) -> assert received["headers"] == {"authorization": "Bearer test-key"} assert received["payload"] == _sample_request().to_json() + assert received["service_peer_name"] == "tepp" From 83a0206698135bb47a977519585bb0d351a179ee Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 22:54:08 +0900 Subject: [PATCH 05/29] fix: harden OpenTelemetry failure boundaries --- backend/app/analysis_run_outbox.py | 2 +- backend/app/main.py | 111 ++++++++++-------- backend/app/post_content_worker.py | 10 +- docker-compose.yml | 1 - lineageweave/http_client.py | 6 +- lineageweave/observability.py | 45 +++++-- tests/test_post_content_worker.py | 4 +- .../test_server_diagnostics.py | 0 8 files changed, 111 insertions(+), 68 deletions(-) rename {backend/tests => tests}/test_server_diagnostics.py (100%) diff --git a/backend/app/analysis_run_outbox.py b/backend/app/analysis_run_outbox.py index fff121cc7..260f540a0 100644 --- a/backend/app/analysis_run_outbox.py +++ b/backend/app/analysis_run_outbox.py @@ -73,7 +73,7 @@ async def publish_outbox_event( try: with traced( "lineageweave.valkey.analysis_outbox_xadd", - {"db.system": "redis", "db.operation.name": "xadd", "lineageweave.stream.kind": "analysis_outbox", "lineageweave.work_kind": work_kind_code}, + {"db.system": "redis", "db.operation.name": "xadd", "lineageweave.stream.kind": "analysis_outbox"}, ): entry_id = await client.xadd( OUTBOX_STREAM_KEY, diff --git a/backend/app/main.py b/backend/app/main.py index b6f983258..dea6b6d13 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -201,44 +201,54 @@ async def lifespan(app: FastAPI): """Open one asyncpg pool and one Valkey client for the process, and close both on shutdown.""" configure_telemetry("lineageweave") - settings = load_settings() - app.state.pool = await create_pool(settings.database_url) - app.state.valkey = create_valkey_client(settings.valkey_url) - app.state.analysis_run_worker = asyncio.create_task( - run_analysis_run_worker( - app.state.valkey, - app.state.pool, - tepp_client=configured_tepp_client( - settings.tepp_transport_url, - settings.tepp_api_key, - ), - adjudication_client=_adjudication_client(), + pool = None + valkey = None + analysis_worker = None + content_worker = None + try: + settings = load_settings() + pool = await create_pool(settings.database_url) + app.state.pool = pool + valkey = create_valkey_client(settings.valkey_url) + app.state.valkey = valkey + analysis_worker = asyncio.create_task( + run_analysis_run_worker( + valkey, + pool, + tepp_client=configured_tepp_client( + settings.tepp_transport_url, + settings.tepp_api_key, + ), + adjudication_client=_adjudication_client(), + ) ) - ) - app.state.post_content_worker = asyncio.create_task( - run_post_content_worker( - app.state.valkey, - app.state.pool, - vision_factory=_vision_client, - embedding_factory=_embedding_client, - structure_factory=_post_structure_client, + app.state.analysis_run_worker = analysis_worker + content_worker = asyncio.create_task( + run_post_content_worker( + valkey, + pool, + vision_factory=_vision_client, + embedding_factory=_embedding_client, + structure_factory=_post_structure_client, + ) ) - ) - try: + app.state.post_content_worker = content_worker yield finally: - app.state.analysis_run_worker.cancel() - app.state.post_content_worker.cancel() - await asyncio.gather( - app.state.analysis_run_worker, - app.state.post_content_worker, - return_exceptions=True, - ) + workers = tuple(worker for worker in (analysis_worker, content_worker) if worker is not None) + for worker in workers: + worker.cancel() + if workers: + await asyncio.gather(*workers, return_exceptions=True) try: - await app.state.pool.close() - await app.state.valkey.aclose() + if pool is not None: + await pool.close() finally: - shutdown_telemetry() + try: + if valkey is not None: + await valkey.aclose() + finally: + shutdown_telemetry() app = FastAPI(title="LineageWeave API", lifespan=lifespan) @@ -2584,25 +2594,30 @@ async def chat_about_post( "cited_posts": stored["cited_posts"], "source_post_ids": source_ids, } - with use_llm_metadata(post_metadata): - client = _post_chat_client() - if not client.available: - record_server_failure( - "post_chat", - RuntimeError("orchestrator unavailable"), - outcome="provider_unavailable", - ) - raise HTTPException( - status.HTTP_503_SERVICE_UNAVAILABLE, - "Post chat is temporarily unavailable. " - "Saved evidence is still available.", - ) - sources = await gather_chat_sources( - conn, post_id, lambda row: _can_see_post(account, row), vision_client=_vision_client() + with use_llm_metadata(post_metadata): + client = _post_chat_client() + if not client.available: + record_server_failure( + "post_chat", + RuntimeError("orchestrator unavailable"), + outcome="provider_unavailable", + ) + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + "Post chat is temporarily unavailable. " + "Saved evidence is still available.", ) try: - with traced("lineageweave.api.post_chat", {"operation_code": "post_chat"}): + async with pool.acquire() as conn: with use_llm_metadata(post_metadata): + sources = await gather_chat_sources( + conn, + post_id, + lambda row: _can_see_post(account, row), + vision_client=_vision_client(), + ) + with use_llm_metadata(post_metadata): + with traced("lineageweave.api.post_chat", {"operation_code": "post_chat"}): answer = await asyncio.to_thread(client.answer, question, sources) except (HttpClientError, KeyError, OSError, TypeError, ValueError) as exc: record_server_failure("post_chat", exc, outcome="provider_unavailable") diff --git a/backend/app/post_content_worker.py b/backend/app/post_content_worker.py index 37aaa9184..e4233df63 100644 --- a/backend/app/post_content_worker.py +++ b/backend/app/post_content_worker.py @@ -3,7 +3,6 @@ from __future__ import annotations import asyncio -import logging import time from collections.abc import Callable from uuid import UUID @@ -16,7 +15,7 @@ from lineageweave.llm_context import build_post_llm_metadata, use_llm_metadata from lineageweave.post_content_normalization import normalize_post_body from lineageweave.post_content_persistence import persist_post_content -from lineageweave.observability import traced +from lineageweave.observability import record_server_failure, traced from lineageweave.post_structure import PostStructureClient from backend.app.config import load_settings @@ -34,7 +33,6 @@ republish_queued_post_content_jobs, ) -_logger = logging.getLogger(__name__) _RECOVERY_INTERVAL_SECONDS = 30.0 _INCOMPLETE_FAILURE_CODE = "post_content_ingestion_incomplete" _ATTEMPT_LIMIT_FAILURE_CODE = "post_content_ingestion_attempt_limit" @@ -66,7 +64,7 @@ async def _claim_job( async with pool.acquire() as conn: async with conn.transaction(): row = await conn.fetchrow( - f""" + """ select p.*, j.source_body_sha256 as job_source_body_sha256, j.status_code as job_status_code, j.attempt_count as job_attempt_count, @@ -271,8 +269,8 @@ async def process_post_content_job( expected_attempt_count=attempt_count, ) return - except Exception: # noqa: BLE001 - durable failure is recorded for retry. - _logger.exception("post content ingestion failed") + except Exception as exc: # noqa: BLE001 - durable failure is recorded for retry. + record_server_failure("post_content_ingestion", exc, outcome="internal_error") await _finish_failed_job( pool, post_id, diff --git a/docker-compose.yml b/docker-compose.yml index 10d81800c..a084426bd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -118,7 +118,6 @@ services: CONTEXTUAL_ORCHESTRATOR_MAX_BODY_BYTES: ${CONTEXTUAL_ORCHESTRATOR_MAX_BODY_BYTES:-8388608} CONTEXTUAL_ORCHESTRATOR_ALLOWED_PROVIDER_HOSTS: ${CONTEXTUAL_ORCHESTRATOR_ALLOWED_PROVIDER_HOSTS:-host.docker.internal} OTEL_SERVICE_NAME: ${OTEL_ORCHESTRATOR_SERVICE_NAME:-contextual-orchestrator} - OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-} command: ["python", "/app/start.py"] ports: - "${ORCHESTRATOR_PORT:-18000}:8000" diff --git a/lineageweave/http_client.py b/lineageweave/http_client.py index e1428b726..d10832a34 100644 --- a/lineageweave/http_client.py +++ b/lineageweave/http_client.py @@ -127,8 +127,7 @@ def post_json( request_payload["metadata"] = {**existing_metadata, **request_metadata} else: raise ValueError("metadata must be an object") - parsed = urlparse(url) - hostname = parsed.hostname or url + hostname = urlparse(url).hostname or url request_headers = {"content-type": "application/json", **headers} session_id = current_session_id() if session_id: @@ -137,8 +136,7 @@ def post_json( "lineageweave.http.post_json", { "http.request.method": "POST", - "server.address": hostname, - "url.path": parsed.path or "/", + "lineageweave.operation_code": "http_post_json", "service.peer.name": service_peer_name, }, ) as span: diff --git a/lineageweave/observability.py b/lineageweave/observability.py index c8f41dc32..6ea0a1185 100644 --- a/lineageweave/observability.py +++ b/lineageweave/observability.py @@ -32,6 +32,33 @@ _TRACE_PROVIDER: Any = None _METER_PROVIDER: Any = None _SERVER_FAILURE_OUTCOMES = {"provider_unavailable", "internal_error"} +_ALLOWED_ATTRIBUTE_KEYS = frozenset( + { + "db.operation.name", + "db.system", + "http.request.method", + "http.response.status_code", + "lineageweave.error_type", + "lineageweave.failure_outcome", + "lineageweave.operation_code", + "lineageweave.session_id", + "lineageweave.stream.kind", + "service.peer.name", + } +) +_ALLOWED_OPERATION_CODES = frozenset( + {"global_ask", "http_post_json", "post_chat", "post_content_ingestion", "unknown"} +) + + +def _bounded_session_id(value: object) -> str | None: + """Return a short, printable session correlation value or ``None``.""" + if not isinstance(value, str): + return None + value = value.strip() + if not value or any(ord(character) < 32 for character in value): + return None + return value[:128] def _otlp_trace_endpoint(endpoint: str) -> str: @@ -59,7 +86,7 @@ def current_session_id() -> str | None: metadata = current_llm_metadata() or {} value = metadata.get("lineageweave_post_session_id") or metadata.get("session_id") - return value if isinstance(value, str) and value else None + return _bounded_session_id(value) def inject_trace_context(carrier: dict[str, str]) -> None: @@ -74,11 +101,13 @@ def _safe_attributes( """Keep telemetry attributes scalar, bounded, and explicitly non-content.""" result: dict[str, str | int | float | bool] = {} for key, value in (attributes or {}).items(): - if ( - not isinstance(key, str) - or not key - or isinstance(value, (dict, list, tuple, set)) - ): + if key not in _ALLOWED_ATTRIBUTE_KEYS: + continue + if key == "lineageweave.session_id": + value = _bounded_session_id(value) + if value is None: + continue + if isinstance(value, (dict, list, tuple, set)): continue if isinstance(value, str): result[key] = value[:256] @@ -223,7 +252,9 @@ def record_server_failure( """ if outcome not in _SERVER_FAILURE_OUTCOMES: raise ValueError(f"unsupported server failure outcome: {outcome}") - bounded_operation = operation_code.strip()[:64] + bounded_operation = operation_code.strip() if isinstance(operation_code, str) else "unknown" + if bounded_operation not in _ALLOWED_OPERATION_CODES: + bounded_operation = "unknown" error_type = type(exc).__name__[:128] session_id = current_session_id() or "" counter = _failure_counter() diff --git a/tests/test_post_content_worker.py b/tests/test_post_content_worker.py index 8ddb16628..2314852fc 100644 --- a/tests/test_post_content_worker.py +++ b/tests/test_post_content_worker.py @@ -193,7 +193,8 @@ async def incomplete(*_args, **_kwargs): assert any(args[1] == QUEUED and args[6] == "post_content_ingestion_incomplete" for args in updates) -def test_transient_provider_error_is_requeued_before_attempt_limit(monkeypatch) -> None: +def test_transient_provider_error_is_requeued_before_attempt_limit(monkeypatch, caplog) -> None: + caplog.set_level("ERROR") connection = _Connection(values=[2]) pool = _Pool(connection) @@ -236,6 +237,7 @@ async def persist(*_args, **_kwargs): for args in updates ) assert all("provider timeout" not in str(args) for args in updates) + assert "provider timeout" not in caplog.text def test_failure_at_attempt_limit_is_terminal_and_visible() -> None: diff --git a/backend/tests/test_server_diagnostics.py b/tests/test_server_diagnostics.py similarity index 100% rename from backend/tests/test_server_diagnostics.py rename to tests/test_server_diagnostics.py From 58cfc4a30590b9fbcc3b8e491daf1fb6a317c972 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 22:56:38 +0900 Subject: [PATCH 06/29] fix: ignore unsupported telemetry attribute keys --- lineageweave/observability.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lineageweave/observability.py b/lineageweave/observability.py index 6ea0a1185..5b5801662 100644 --- a/lineageweave/observability.py +++ b/lineageweave/observability.py @@ -101,7 +101,7 @@ def _safe_attributes( """Keep telemetry attributes scalar, bounded, and explicitly non-content.""" result: dict[str, str | int | float | bool] = {} for key, value in (attributes or {}).items(): - if key not in _ALLOWED_ATTRIBUTE_KEYS: + if not isinstance(key, str) or key not in _ALLOWED_ATTRIBUTE_KEYS: continue if key == "lineageweave.session_id": value = _bounded_session_id(value) From b0c9bc6466c77b9b53f5e0e17224c868bfe08be6 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 23:06:41 +0900 Subject: [PATCH 07/29] test: enforce printable telemetry session ids --- lineageweave/observability.py | 2 +- tests/test_observability.py | 26 ++++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/lineageweave/observability.py b/lineageweave/observability.py index 5b5801662..4404fb6b6 100644 --- a/lineageweave/observability.py +++ b/lineageweave/observability.py @@ -56,7 +56,7 @@ def _bounded_session_id(value: object) -> str | None: if not isinstance(value, str): return None value = value.strip() - if not value or any(ord(character) < 32 for character in value): + if not value or not value.isprintable(): return None return value[:128] diff --git a/tests/test_observability.py b/tests/test_observability.py index 4ae04509b..5c2c74998 100644 --- a/tests/test_observability.py +++ b/tests/test_observability.py @@ -3,6 +3,8 @@ from lineageweave import http_client, observability from lineageweave.llm_context import use_llm_metadata from lineageweave.observability import ( + _bounded_session_id, + _safe_attributes, _otlp_trace_endpoint, current_session_id, shutdown_telemetry, @@ -42,6 +44,30 @@ def test_current_session_id_reads_existing_context(): assert current_session_id() == "post-session-2" +def test_session_correlation_is_printable_and_bounded(): + """Session correlation rejects controls and caps values before export.""" + assert _bounded_session_id(" session-3 ") == "session-3" + assert _bounded_session_id("session\n3") is None + assert _bounded_session_id("session\x7f3") is None + assert _bounded_session_id("x" * 129) == "x" * 128 + assert _bounded_session_id(3) is None + + +def test_safe_attributes_drops_unknown_keys_and_invalid_session_values(): + """Telemetry keeps only the allowlisted scalar boundary attributes.""" + assert _safe_attributes( + { + "request.path": "/private/synthetic", + "lineageweave.session_id": "bad\nvalue", + "service.peer.name": "tepp", + "http.response.status_code": 503, + } + ) == { + "service.peer.name": "tepp", + "http.response.status_code": 503, + } + + def test_traced_rethrows_provider_errors(): """Observability never converts a failed provider operation into success.""" try: From 720004942dd155a85020af32da402d320038f46a Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Fri, 21 Aug 2026 23:30:59 +0900 Subject: [PATCH 08/29] fix(otel): retain endpoint operation diagnostics --- backend/app/main.py | 10 ++++++++-- lineageweave/observability.py | 3 ++- tests/test_observability.py | 9 +++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/backend/app/main.py b/backend/app/main.py index dea6b6d13..37c51e8ae 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -2617,7 +2617,10 @@ async def chat_about_post( vision_client=_vision_client(), ) with use_llm_metadata(post_metadata): - with traced("lineageweave.api.post_chat", {"operation_code": "post_chat"}): + with traced( + "lineageweave.api.post_chat", + {"lineageweave.operation_code": "post_chat"}, + ): answer = await asyncio.to_thread(client.answer, question, sources) except (HttpClientError, KeyError, OSError, TypeError, ValueError) as exc: record_server_failure("post_chat", exc, outcome="provider_unavailable") @@ -2692,7 +2695,10 @@ async def ask_agent( "next_action": "No authorized source posts are available for this question.", } try: - with traced("lineageweave.api.global_ask", {"operation_code": "global_ask"}): + with traced( + "lineageweave.api.global_ask", + {"lineageweave.operation_code": "global_ask"}, + ): answer = await asyncio.to_thread(client.answer, question, sources) except (HttpClientError, KeyError, OSError, TypeError, ValueError) as exc: record_server_failure("global_ask", exc, outcome="provider_unavailable") diff --git a/lineageweave/observability.py b/lineageweave/observability.py index 4404fb6b6..07f5c9ca4 100644 --- a/lineageweave/observability.py +++ b/lineageweave/observability.py @@ -176,7 +176,7 @@ def configure_telemetry(service_name: str = "lineageweave") -> None: def shutdown_telemetry() -> None: """Flush configured OTLP providers without masking application shutdown.""" - global _TRACE_PROVIDER, _METER_PROVIDER, _FAILURE_COUNTER + global _CONFIGURED, _TRACE_PROVIDER, _METER_PROVIDER, _FAILURE_COUNTER for provider_name, provider in ( ("trace", _TRACE_PROVIDER), ("metric", _METER_PROVIDER), @@ -193,6 +193,7 @@ def shutdown_telemetry() -> None: _TRACE_PROVIDER = None _METER_PROVIDER = None _FAILURE_COUNTER = None + _CONFIGURED = False def _failure_counter() -> Any: diff --git a/tests/test_observability.py b/tests/test_observability.py index 5c2c74998..6f263cd23 100644 --- a/tests/test_observability.py +++ b/tests/test_observability.py @@ -79,6 +79,13 @@ def test_traced_rethrows_provider_errors(): raise AssertionError("traced must preserve operation failures") +def test_safe_attributes_keeps_allowlisted_operation_code(): + """Endpoint spans retain the bounded operation dimension.""" + assert _safe_attributes( + {"lineageweave.operation_code": "post_chat"} + ) == {"lineageweave.operation_code": "post_chat"} + + def test_otlp_base_endpoint_gets_trace_signal_path(): """A configured collector base URL receives the HTTP traces signal path.""" assert _otlp_trace_endpoint("http://collector:4318") == "http://collector:4318/v1/traces" @@ -99,6 +106,7 @@ def shutdown(self): monkeypatch.setattr(observability, "_TRACE_PROVIDER", _Provider("trace")) monkeypatch.setattr(observability, "_METER_PROVIDER", _Provider("metric")) monkeypatch.setattr(observability, "_FAILURE_COUNTER", object()) + monkeypatch.setattr(observability, "_CONFIGURED", True) shutdown_telemetry() @@ -106,3 +114,4 @@ def shutdown(self): assert observability._TRACE_PROVIDER is None assert observability._METER_PROVIDER is None assert observability._FAILURE_COUNTER is None + assert observability._CONFIGURED is False From 7a79cfa21717281138e6f1eb714e26e220f52e4e Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Sat, 22 Aug 2026 00:28:52 +0900 Subject: [PATCH 09/29] fix(frontend): keep admin controls behind authentication --- frontend/src/App.tsx | 2 -- 1 file changed, 2 deletions(-) diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 6fba0dd41..666888a4d 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -101,7 +101,6 @@ import { tf, useLocale, } from "./i18n"; -import { rememberOidcReturnUrl, returnUrlFromLocation } from "./oidcReturnUrl"; import "./App.css"; function orchestratorUnavailableMessage(err: unknown, action: string): string { @@ -4620,7 +4619,6 @@ export default function App({ showLabPanels = false }: { showLabPanels?: boolean Enterprise SSO Authentication - {destination === "admin" ? : null}