diff --git a/AGENTS.md b/AGENTS.md index bc8768ced..74e83f0e8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -145,6 +145,18 @@ contextual-orchestrator owns model discovery and selection. caption and OCR (ADR 0155); never show an internal LLM instruction such as `This post is an image` to a buyer. +## Observability boundary + +- Follow governance-risk-compliance ADR 0009 and LineageWeave ADR 0122 for + OpenTelemetry. Use `OTEL_SERVICE_NAME` and + `OTEL_EXPORTER_OTLP_ENDPOINT`; exporting is opt-in and provider-neutral. +- Correlate one post's HTTP, contextual-orchestrator, and Valkey work with the + existing post-scoped session metadata. Do not create an ad hoc session table. +- Telemetry may contain bounded operation, route-template, service-peer, and + correlation attributes, but never post body, prompt, answer, source content, + actor or tenant identifiers, credentials, raw stream keys, or provider + responses. GRC remains the control/evidence owner. + ## Source parsing and semantic units - Preserve the source representation and provenance, then derive semantic diff --git a/CHANGELOG.d/2.13.2-otel-server-diagnostics.md b/CHANGELOG.d/2.13.2-otel-server-diagnostics.md new file mode 100644 index 000000000..ef5953bbe --- /dev/null +++ b/CHANGELOG.d/2.13.2-otel-server-diagnostics.md @@ -0,0 +1,6 @@ +## 2.13.2 + +- Add reader-safe Global Ask and post-chat failures with bounded OpenTelemetry + metrics, traces, and structured server diagnostics for GRC consumption. + Failure logs carry the active TraceId and SpanId so another agent can join + the Error span to the audit record. diff --git a/CHANGELOG.md b/CHANGELOG.md index f813967d8..4216b90cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -147,6 +147,15 @@ All notable changes to this project are documented here. Format follows is unwired, with `이 범위의 일정을 아직 받을 수 없습니다`. Weekly VOC and newspaper stay on the board. +## [2.13.2] - 2026-08-24 + +### Added + +- Reader-safe Global Ask and post-chat failures now carry bounded + OpenTelemetry metrics, traces, and structured server diagnostics for + GRC consumption. Failure logs carry the active TraceId and SpanId so + another agent can join the Error span to the audit record. + ## [2.12.26] - 2026-08-24 ### Added diff --git a/backend/app/activity_stream.py b/backend/app/activity_stream.py index a47bf9343..56311382b 100644 --- a/backend/app/activity_stream.py +++ b/backend/app/activity_stream.py @@ -14,6 +14,8 @@ import redis.asyncio as redis from fastapi import Request +from lineageweave.observability import traced + def create_valkey_client(url: str) -> redis.Redis: """One shared async client for the process, mirroring db.create_pool.""" @@ -66,12 +68,16 @@ async def publish_activity_event( ``approximate=True``) so one very active post's stream can't grow without bound -- the panel only ever shows the most recent 50 anyway. """ - return await client.xadd( - _stream_key(post_id), - _activity_fields(event_type, actor_account_id, summary), - maxlen=1000, - approximate=True, - ) + with traced( + "lineageweave.valkey.activity_xadd", + {"db.system": "redis", "db.operation.name": "xadd", "lineageweave.stream.kind": "activity"}, + ): + return await client.xadd( + _stream_key(post_id), + _activity_fields(event_type, actor_account_id, summary), + maxlen=1000, + approximate=True, + ) def publish_activity_event_sync( @@ -83,20 +89,32 @@ def publish_activity_event_sync( ) -> str | None: """Sync ``XADD`` for ``make seed``. Returns None if ``summary`` is already on the stream.""" key = _stream_key(post_id) - existing = client.xrevrange(key, count=50) + with traced( + "lineageweave.valkey.activity_xrevrange", + {"db.system": "redis", "db.operation.name": "xrevrange", "lineageweave.stream.kind": "activity"}, + ): + existing = client.xrevrange(key, count=50) if any(fields.get("summary") == summary for _entry_id, fields in existing): return None - return client.xadd( - key, - _activity_fields(event_type, str(actor_account_id), summary), - maxlen=1000, - approximate=True, - ) + with traced( + "lineageweave.valkey.activity_xadd", + {"db.system": "redis", "db.operation.name": "xadd", "lineageweave.stream.kind": "activity"}, + ): + return client.xadd( + key, + _activity_fields(event_type, str(actor_account_id), summary), + maxlen=1000, + approximate=True, + ) async def read_activity_events(client: redis.Redis, post_id: str, count: int = 50) -> list[dict[str, Any]]: """The post's most recent events, newest first.""" - entries = await client.xrevrange(_stream_key(post_id), count=count) + with traced( + "lineageweave.valkey.activity_xrevrange", + {"db.system": "redis", "db.operation.name": "xrevrange", "lineageweave.stream.kind": "activity"}, + ): + entries = await client.xrevrange(_stream_key(post_id), count=count) return [ { "event_id": entry_id, diff --git a/backend/app/analysis_run_outbox.py b/backend/app/analysis_run_outbox.py index 487948ba8..260f540a0 100644 --- a/backend/app/analysis_run_outbox.py +++ b/backend/app/analysis_run_outbox.py @@ -13,6 +13,8 @@ import redis.asyncio as redis +from lineageweave.observability import traced + OUTBOX_STREAM_KEY = "analysis-run-outbox" _CLAIMED = "analysis_outbox_claimed" _DELIVERED = "analysis_outbox_delivered" @@ -69,16 +71,20 @@ async def publish_outbox_event( if client is None: return None try: - entry_id = await client.xadd( - OUTBOX_STREAM_KEY, - outbox_stream_fields( - analysis_run_id=analysis_run_id, - work_kind_code=work_kind_code, - request_sha256=request_sha256, - ), - maxlen=1000, - approximate=True, - ) + with traced( + "lineageweave.valkey.analysis_outbox_xadd", + {"db.system": "redis", "db.operation.name": "xadd", "lineageweave.stream.kind": "analysis_outbox"}, + ): + entry_id = await client.xadd( + OUTBOX_STREAM_KEY, + outbox_stream_fields( + analysis_run_id=analysis_run_id, + work_kind_code=work_kind_code, + request_sha256=request_sha256, + ), + maxlen=1000, + approximate=True, + ) except redis.RedisError: return None return str(entry_id) diff --git a/backend/app/analysis_run_start.py b/backend/app/analysis_run_start.py index 9e097e0fb..54a50f733 100644 --- a/backend/app/analysis_run_start.py +++ b/backend/app/analysis_run_start.py @@ -137,7 +137,13 @@ def transport(payload: dict[str, Any]) -> dict[str, Any]: """POST the TEPP wire payload to `url`, raising TeppNotAvailable on any transport failure.""" try: headers = {"authorization": f"Bearer {api_key}"} if api_key.strip() else {} - return post_json(url, payload, headers=headers, timeout=30.0) + return post_json( + url, + payload, + headers=headers, + timeout=30.0, + service_peer_name="tepp", + ) except (HttpClientError, OSError, ValueError, TypeError) as exc: # Chain internally for operator logging; the exposed # message stays generic, never the raw provider exception text. diff --git a/backend/app/analysis_run_worker.py b/backend/app/analysis_run_worker.py index 166810e13..4f3d76ead 100644 --- a/backend/app/analysis_run_worker.py +++ b/backend/app/analysis_run_worker.py @@ -7,6 +7,7 @@ from __future__ import annotations +import asyncio import logging import asyncpg @@ -14,13 +15,15 @@ from uuid import UUID from lineageweave.adjudication_client import AdjudicationClient +from lineageweave.observability import traced from lineageweave.tepp_client import TeppClient from backend.app.analysis_run_ingestion import AnalysisRunCreateError from backend.app.analysis_run_outbox import OUTBOX_STREAM_KEY from backend.app.analysis_run_start import deliver_queued_analysis_run -logger = logging.getLogger(__name__) +_BROKER_RECOVERY_DELAY_SECONDS = 1.0 +_worker_logger = logging.getLogger(__name__) async def consume_analysis_run_stream_once( @@ -36,50 +39,72 @@ async def consume_analysis_run_stream_once( Invalid or stale entries are acknowledged by advancing the cursor; the durable PostgreSQL outbox remains available for a later explicit retry. """ - batches = await client.xread({OUTBOX_STREAM_KEY: last_id}, count=10, block=1000) - for _stream_name, entries in batches: - for entry_id, fields in entries: - analysis_run_id = str(fields.get("analysis_run_id", "")).strip() - try: - UUID(analysis_run_id) - except ValueError: - analysis_run_id = "" - if analysis_run_id: - # One run's fail-closed refusal (404/409/503, e.g. channel - # weights not estimated yet, ADR 0145) must not end the - # worker task and halt every later run's delivery. The - # transaction rolls back, the durable outbox row stays - # available, and an explicit HTTP start retries the run - # once the operator resolves the named next action. + try: + batches = await client.xread({OUTBOX_STREAM_KEY: last_id}, count=10, block=1000) + except Exception: + # Keep idle polls silent, but retain a diagnostic span for broker failures. + with traced( + "lineageweave.valkey.analysis_outbox_xread", + { + "db.system": "redis", + "db.operation.name": "xread", + "lineageweave.stream.kind": "analysis_outbox", + }, + ): + raise + if not batches: + return last_id + with traced( + "lineageweave.valkey.analysis_outbox_batch", + { + "db.system": "redis", + "db.operation.name": "xread", + "lineageweave.stream.kind": "analysis_outbox", + }, + ): + for _stream_name, entries in batches: + for entry_id, fields in entries: + analysis_run_id = str(fields.get("analysis_run_id", "")).strip() try: - async with pool.acquire() as conn: - async with conn.transaction(): - owner = await conn.fetchrow( - """ - select requested_by_account_id - from analysis_run - where analysis_run_id = $1::uuid - """, - analysis_run_id, - ) - if owner is not None: - await deliver_queued_analysis_run( - conn, - analysis_run_id=analysis_run_id, - account_id=str(owner["requested_by_account_id"]), - affiliated_entity_ids=[], - tepp_client=tepp_client, - adjudication_client=adjudication_client, - valkey_stream_entry_id=str(entry_id), + UUID(analysis_run_id) + except ValueError: + analysis_run_id = "" + if analysis_run_id: + # One run's fail-closed refusal (404/409/503, e.g. channel + # weights not estimated yet, ADR 0145) must not end the + # worker task and halt every later run's delivery. The + # transaction rolls back, the durable outbox row stays + # available, and an explicit HTTP start retries the run + # once the operator resolves the named next action. + try: + async with pool.acquire() as conn: + async with conn.transaction(): + owner = await conn.fetchrow( + """ + select requested_by_account_id + from analysis_run + where analysis_run_id = $1::uuid + """, + analysis_run_id, ) - except AnalysisRunCreateError as exc: - logger.warning( - "analysis-run %s delivery refused (%s): %s", - analysis_run_id, - exc.status_code, - exc.detail, - ) - last_id = str(entry_id) + if owner is not None: + await deliver_queued_analysis_run( + conn, + analysis_run_id=analysis_run_id, + account_id=str(owner["requested_by_account_id"]), + affiliated_entity_ids=[], + tepp_client=tepp_client, + adjudication_client=adjudication_client, + valkey_stream_entry_id=str(entry_id), + ) + except AnalysisRunCreateError as exc: + _worker_logger.warning( + "analysis-run %s delivery refused (%s): %s", + analysis_run_id, + exc.status_code, + exc.detail, + ) + last_id = str(entry_id) return last_id @@ -93,10 +118,16 @@ async def run_analysis_run_worker( """Run the single-process wake-up consumer until task cancellation.""" last_id = "0-0" while True: - last_id = await consume_analysis_run_stream_once( - client, - pool, - last_id=last_id, - tepp_client=tepp_client, - adjudication_client=adjudication_client, - ) + try: + last_id = await consume_analysis_run_stream_once( + client, + pool, + last_id=last_id, + tepp_client=tepp_client, + adjudication_client=adjudication_client, + ) + except (redis.RedisError, OSError) as exc: + _worker_logger.warning( + "analysis-run Valkey poll failed; retrying (error_type=%s)", type(exc).__name__ + ) + await asyncio.sleep(_BROKER_RECOVERY_DELAY_SECONDS) diff --git a/backend/app/auth.py b/backend/app/auth.py index 695a43a40..044dbaf52 100644 --- a/backend/app/auth.py +++ b/backend/app/auth.py @@ -48,11 +48,15 @@ def _jwks(settings: Settings, *, force_refresh: bool = False) -> dict: if settings.oidc_jwks_uri_override: jwks_uri = settings.oidc_jwks_uri_override else: - metadata = get_json(settings.oidc_discovery_uri, timeout=10) + metadata = get_json( + settings.oidc_discovery_uri, + timeout=10, + service_peer_name="oidc", + ) jwks_uri = metadata.get("jwks_uri") if not isinstance(jwks_uri, str) or not jwks_uri.strip(): raise ValueError("OIDC discovery document has no jwks_uri") - cached = get_json(jwks_uri, timeout=10) + cached = get_json(jwks_uri, timeout=10, service_peer_name="oidc") except (HttpClientError, OSError, ValueError) as exc: raise HTTPException( status.HTTP_503_SERVICE_UNAVAILABLE, diff --git a/backend/app/global_ask_queue.py b/backend/app/global_ask_queue.py index ce638efac..4332a596f 100644 --- a/backend/app/global_ask_queue.py +++ b/backend/app/global_ask_queue.py @@ -20,26 +20,31 @@ import asyncio import logging import time +from collections.abc import Callable from datetime import date -from typing import Any, Callable +from typing import Any import asyncpg import redis.asyncio as redis from fastapi import HTTPException, status from lineageweave.http_client import HttpClientError +from lineageweave.observability import record_server_failure from lineageweave.post_chat import ( PostChatClient, cited_post_evidence, cited_post_summaries, ) - from lineageweave.temporal_expressions import resolve_korean_relative_time from .config import GLOBAL_ASK_JOB_DEADLINE_SECONDS from .lineage_ingestion import lineage_graphs_for_posts from .operability import log_internal_fault, log_provider_unavailable -from .post_chat_ingestion import _seoul_today, cited_post_images, gather_global_chat_sources +from .post_chat_ingestion import ( + _seoul_today, + cited_post_images, + gather_global_chat_sources, +) GLOBAL_ASK_STREAM_KEY = "global_ask_request_stream" @@ -167,14 +172,22 @@ def can_see(row: asyncpg.Record) -> bool: return str(row["corporate_entity_id"]) in corporate_entity_ids today = _seoul_today() - async with pool.acquire() as conn: - sources = await gather_global_chat_sources( - conn, - can_see, - corporate_entity_ids, - question=question_text, - today=today, - ) + try: + async with pool.acquire() as conn: + sources = await gather_global_chat_sources( + conn, + can_see, + corporate_entity_ids, + question=question_text, + today=today, + ) + except Exception as exc: + log_internal_fault("global_ask", exc) + record_server_failure("global_ask", exc, outcome="internal_error") + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + "Ask Agent is unavailable: authorized evidence could not be assembled", + ) from exc if not sources: return { "answer_text": "", @@ -195,6 +208,7 @@ def can_see(row: asyncpg.Record) -> bool: # failure path so callers cannot probe which internal classifier # fired; the event_type distinction lives only in server logs. log_provider_unavailable("global_ask", exc) + record_server_failure("global_ask", exc, outcome="provider_unavailable") raise HTTPException( status.HTTP_503_SERVICE_UNAVAILABLE, "Ask Agent is unavailable: contextual-orchestrator could not complete the answer", @@ -203,6 +217,7 @@ def can_see(row: asyncpg.Record) -> bool: # Contract/schema fault: the orchestrator responded but its payload # did not match the evidence-object contract. log_internal_fault("global_ask", exc) + record_server_failure("global_ask", exc, outcome="provider_unavailable") raise HTTPException( status.HTTP_503_SERVICE_UNAVAILABLE, "Ask Agent is unavailable: contextual-orchestrator could not complete the answer", @@ -211,6 +226,7 @@ def can_see(row: asyncpg.Record) -> bool: # Unexpected defect. Keep the customer boundary and emit a full # structured internal-fault diagnostic (message-redacted). log_internal_fault("global_ask", exc) + record_server_failure("global_ask", exc, outcome="internal_error") raise HTTPException( status.HTTP_503_SERVICE_UNAVAILABLE, "Ask Agent is unavailable: contextual-orchestrator could not complete the answer", @@ -308,7 +324,7 @@ async def process_global_ask_job( # Shutdown: leave the row `running`; the recovery sweep re-queues # it after the orphan window on the next process start. raise - except Exception as exc: # noqa: BLE001 - settlement must be fail-closed + except Exception as exc: # A narrow exception tuple here once let an unexpected error kill # the task silently and strand the row `running` until orphan # recovery (observed live) — every failure settles the job. @@ -500,7 +516,7 @@ async def run_global_ask_worker( ) except asyncio.CancelledError: raise - except Exception: # noqa: BLE001 - one bad round must not stop Ask + except Exception: _logger.exception("global ask worker round failed; retrying") await asyncio.sleep(5) finally: diff --git a/backend/app/main.py b/backend/app/main.py index 572d93ca1..bce2a716c 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -53,6 +53,12 @@ from lineageweave.image_content import orchestrator_vision_client from lineageweave.embedding_client import orchestrator_embedding_client from lineageweave.llm_context import build_post_llm_metadata, use_llm_metadata +from lineageweave.observability import ( + configure_telemetry, + record_server_failure, + shutdown_telemetry, + traced, +) from lineageweave.corporate_hierarchy_inference import ( ContextualOrchestratorHierarchyInferenceClient, NullCorporateHierarchyInferenceClient, @@ -217,56 +223,74 @@ async def lifespan(app: FastAPI): """Open one asyncpg pool and one Valkey client for the process, and close both on shutdown.""" - settings = load_settings() - app.state.pool = await create_pool(settings.database_url) - app.state.valkey = create_valkey_client(settings.valkey_url) - app.state.analysis_run_worker = asyncio.create_task( - run_analysis_run_worker( - app.state.valkey, - app.state.pool, - tepp_client=configured_tepp_client( - settings.tepp_transport_url, - settings.tepp_api_key, - ), - adjudication_client=_adjudication_client(), + configure_telemetry("lineageweave") + pool = None + valkey = None + analysis_worker = None + content_worker = None + global_ask_worker = None + try: + settings = load_settings() + pool = await create_pool(settings.database_url) + app.state.pool = pool + valkey = create_valkey_client(settings.valkey_url) + app.state.valkey = valkey + analysis_worker = asyncio.create_task( + run_analysis_run_worker( + valkey, + pool, + tepp_client=configured_tepp_client( + settings.tepp_transport_url, + settings.tepp_api_key, + ), + adjudication_client=_adjudication_client(), + ) ) - ) - app.state.post_content_worker = asyncio.create_task( - run_post_content_worker( - app.state.valkey, - app.state.pool, - vision_factory=_vision_client, - embedding_factory=_embedding_client, - structure_factory=_post_structure_client, + app.state.analysis_run_worker = analysis_worker + content_worker = asyncio.create_task( + run_post_content_worker( + valkey, + pool, + vision_factory=_vision_client, + embedding_factory=_embedding_client, + structure_factory=_post_structure_client, + ) ) - ) - # Late-bound lambda so tests that monkeypatch _post_chat_client reach - # the worker too (the name resolves in module globals at call time). - # Only this worker gets the long answer timeout; the per-post chat - # endpoint keeps the client's interactive default. - app.state.global_ask_worker = asyncio.create_task( - run_global_ask_worker( - app.state.valkey, - app.state.pool, - chat_factory=lambda: _post_chat_client( - timeout=load_settings().orchestrator_answer_timeout_seconds - ), + app.state.post_content_worker = content_worker + # Late-bound lambda so tests that monkeypatch _post_chat_client reach + # the worker too (the name resolves in module globals at call time). + # Only this worker gets the long answer timeout; the per-post chat + # endpoint keeps the client's interactive default. + global_ask_worker = asyncio.create_task( + run_global_ask_worker( + valkey, + pool, + chat_factory=lambda: _post_chat_client( + timeout=load_settings().orchestrator_answer_timeout_seconds + ), + ) ) - ) - try: + app.state.global_ask_worker = global_ask_worker yield finally: - app.state.analysis_run_worker.cancel() - app.state.post_content_worker.cancel() - app.state.global_ask_worker.cancel() - await asyncio.gather( - app.state.analysis_run_worker, - app.state.post_content_worker, - app.state.global_ask_worker, - return_exceptions=True, + workers = tuple( + worker + for worker in (analysis_worker, content_worker, global_ask_worker) + if worker is not None ) - await app.state.pool.close() - await app.state.valkey.aclose() + for worker in workers: + worker.cancel() + if workers: + await asyncio.gather(*workers, return_exceptions=True) + try: + if pool is not None: + await pool.close() + finally: + try: + if valkey is not None: + await valkey.aclose() + finally: + shutdown_telemetry() logger = logging.getLogger(__name__) @@ -2362,7 +2386,8 @@ async def evaluate_post( if not client.available: raise HTTPException( status.HTTP_503_SERVICE_UNAVAILABLE, - "Post evaluation is unavailable: set ORCHESTRATOR_BASE_URL / ORCHESTRATOR_API_KEY", + "Post evaluation is unavailable. Ask an administrator to configure the " + "analysis service, then retry.", ) async with pool.acquire() as conn: body_row = await conn.fetchrow("select post_body from source_post where post_id = $1", post_id) @@ -2759,7 +2784,9 @@ async def chat_about_post( """ question = request.question.strip() if not question: - raise HTTPException(status.HTTP_422_UNPROCESSABLE_ENTITY, "question is required") + raise HTTPException( + status.HTTP_422_UNPROCESSABLE_ENTITY, "question is required" + ) post = await _load_visible_post(post_id, account, pool) post_metadata = build_post_llm_metadata(post_id, post) async with pool.acquire() as conn: @@ -2774,29 +2801,55 @@ async def chat_about_post( "cited_posts": stored["cited_posts"], "source_post_ids": source_ids, } - with use_llm_metadata(post_metadata): - client = _post_chat_client() - if not client.available: + with use_llm_metadata(post_metadata): + with traced( + "lineageweave.api.post_chat", + {"lineageweave.operation_code": "post_chat"}, + ): + try: + client = _post_chat_client() + if not client.available: + record_server_failure( + "post_chat", + RuntimeError("orchestrator unavailable"), + outcome="provider_unavailable", + ) + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + "Post chat is temporarily unavailable. " + "Saved evidence is still available.", + ) + async with pool.acquire() as conn: + sources = await gather_chat_sources( + conn, + post_id, + lambda row: _can_see_post(account, row), + vision_client=_vision_client(), + ) + answer = await asyncio.to_thread(client.answer, question, sources) + except HTTPException: + raise + except ( + HttpClientError, + TimeoutError, + KeyError, + OSError, + TypeError, + ValueError, + ) as exc: + record_server_failure("post_chat", exc, outcome="provider_unavailable") raise HTTPException( status.HTTP_503_SERVICE_UNAVAILABLE, - "Post chat is unavailable: set ORCHESTRATOR_BASE_URL / ORCHESTRATOR_API_KEY", - ) - sources = await gather_chat_sources( - conn, post_id, lambda row: _can_see_post(account, row), vision_client=_vision_client() - ) - try: - with use_llm_metadata(post_metadata): - answer = await asyncio.to_thread(client.answer, question, sources) - except (HttpClientError, KeyError, OSError, TypeError, ValueError) as exc: - raise HTTPException( - status.HTTP_503_SERVICE_UNAVAILABLE, - "Post chat is unavailable: contextual-orchestrator returned no complete evidence object", - ) from exc - except Exception as exc: # noqa: BLE001 - provider boundary is fail-closed. - raise HTTPException( - status.HTTP_503_SERVICE_UNAVAILABLE, - "Post chat is unavailable: contextual-orchestrator returned no complete evidence object", - ) from exc + "Post chat is temporarily unavailable. " + "Saved evidence is still available.", + ) from exc + except Exception as exc: + record_server_failure("post_chat", exc, outcome="internal_error") + raise HTTPException( + status.HTTP_503_SERVICE_UNAVAILABLE, + "Post chat is temporarily unavailable. " + "Saved evidence is still available.", + ) from exc cited_ids = list(answer.cited_post_ids) async with pool.acquire() as conn: await persist_post_chat(conn, post_id, question, answer.answer_text, cited_ids) @@ -2839,7 +2892,8 @@ async def ask_agent( if not _post_chat_client().available: raise HTTPException( status.HTTP_503_SERVICE_UNAVAILABLE, - "Ask Agent is unavailable: set ORCHESTRATOR_BASE_URL / ORCHESTRATOR_API_KEY", + "Ask Agent is unavailable. Ask an administrator to configure the analysis service, " + "then retry.", ) async with pool.acquire() as conn: job_id = await enqueue_global_ask_job( diff --git a/backend/app/post_content_queue.py b/backend/app/post_content_queue.py index 29547cfad..9f3fc90f4 100644 --- a/backend/app/post_content_queue.py +++ b/backend/app/post_content_queue.py @@ -10,6 +10,8 @@ import asyncpg import redis.asyncio as redis +from lineageweave.observability import traced + POST_CONTENT_STREAM_KEY = "post-content-ingestion" QUEUED = "post_content_ingestion_queued" RUNNING = "post_content_ingestion_running" @@ -129,15 +131,19 @@ async def publish_post_content_event( if client is None: return None try: - entry_id = await client.xadd( - POST_CONTENT_STREAM_KEY, - post_content_stream_fields( - post_id=post_id, - source_body_digest=source_body_digest, - ), - maxlen=1000, - approximate=True, - ) + with traced( + "lineageweave.valkey.post_content_xadd", + {"db.system": "redis", "db.operation.name": "xadd", "lineageweave.stream.kind": "post_content"}, + ): + entry_id = await client.xadd( + POST_CONTENT_STREAM_KEY, + post_content_stream_fields( + post_id=post_id, + source_body_digest=source_body_digest, + ), + maxlen=1000, + approximate=True, + ) except redis.RedisError: return None return str(entry_id) diff --git a/backend/app/post_content_worker.py b/backend/app/post_content_worker.py index b00d748d5..f1ee2c109 100644 --- a/backend/app/post_content_worker.py +++ b/backend/app/post_content_worker.py @@ -11,13 +11,6 @@ import asyncpg import redis.asyncio as redis -from lineageweave.embedding_client import EmbeddingClient -from lineageweave.image_content import ImageContentClient -from lineageweave.llm_context import build_post_llm_metadata, use_llm_metadata -from lineageweave.post_content_normalization import normalize_post_body -from lineageweave.post_content_persistence import persist_post_content -from lineageweave.post_structure import PostStructureClient - from backend.app.config import load_settings from backend.app.post_content_queue import ( FAILED, @@ -29,12 +22,21 @@ STALE_RUNNING_INTERVAL, SUCCEEDED, post_content_is_complete, - transition_post_content_job, republish_queued_post_content_jobs, + transition_post_content_job, ) +from lineageweave.embedding_client import EmbeddingClient +from lineageweave.http_client import HttpClientError +from lineageweave.image_content import ImageContentClient +from lineageweave.llm_context import build_post_llm_metadata, use_llm_metadata +from lineageweave.observability import record_server_failure, traced +from lineageweave.post_content_normalization import normalize_post_body +from lineageweave.post_content_persistence import persist_post_content +from lineageweave.post_structure import PostStructureClient _logger = logging.getLogger(__name__) _RECOVERY_INTERVAL_SECONDS = 30.0 +_BROKER_RECOVERY_DELAY_SECONDS = 1.0 _INCOMPLETE_FAILURE_CODE = "post_content_ingestion_incomplete" _ATTEMPT_LIMIT_FAILURE_CODE = "post_content_ingestion_attempt_limit" _SOURCE_BODY_MISSING_FAILURE_CODE = "post_content_source_body_missing" @@ -43,7 +45,15 @@ async def _stream_tail(client: redis.Redis) -> str: """Start after historical wake-ups; the normalized ledger drives recovery.""" - rows = await client.xrevrange(POST_CONTENT_STREAM_KEY, count=1) + with traced( + "lineageweave.valkey.post_content_xrevrange", + { + "db.system": "redis", + "db.operation.name": "xrevrange", + "lineageweave.stream.kind": "post_content", + }, + ): + rows = await client.xrevrange(POST_CONTENT_STREAM_KEY, count=1) return str(rows[0][0]) if rows else "0-0" @@ -58,7 +68,7 @@ async def _claim_job( async with pool.acquire() as conn: async with conn.transaction(): row = await conn.fetchrow( - f""" + """ select p.*, j.source_body_sha256 as job_source_body_sha256, j.status_code as job_status_code, j.attempt_count as job_attempt_count, @@ -282,8 +292,16 @@ async def process_post_content_job( expected_attempt_count=attempt_count, ) return - except Exception: # noqa: BLE001 - durable failure is recorded for retry. - _logger.exception("post content ingestion failed for post_id=%s", post_id) + except Exception as exc: # noqa: BLE001 - durable failure is recorded for retry. + _logger.error("post content ingestion failed for post_id=%s", post_id) + outcome = ( + "provider_unavailable" + if isinstance( + exc, (HttpClientError, TimeoutError, KeyError, OSError, ValueError) + ) + else "internal_error" + ) + record_server_failure("post_content_ingestion", exc, outcome=outcome) await _finish_failed_job( pool, post_id, @@ -310,25 +328,47 @@ async def consume_post_content_stream_once( for each, and returns the last-seen entry id so the caller can resume from there on the next poll. """ - batches = await client.xread({POST_CONTENT_STREAM_KEY: last_id}, count=10, block=1000) - for _stream_name, entries in batches: - for entry_id, fields in entries: - post_id = str(fields.get("post_id", "")).strip() - digest = str(fields.get("source_body_sha256", "")).strip() - try: - UUID(post_id) - except ValueError: - post_id = "" - if post_id and len(digest) == 64: - await process_post_content_job( - pool, - post_id=post_id, - source_body_digest=digest, - vision_factory=vision_factory, - embedding_factory=embedding_factory, - structure_factory=structure_factory, - ) - last_id = str(entry_id) + try: + batches = await client.xread({POST_CONTENT_STREAM_KEY: last_id}, count=10, block=1000) + except Exception: + # Keep idle polls silent, but retain a diagnostic span for broker failures. + with traced( + "lineageweave.valkey.post_content_xread", + { + "db.system": "redis", + "db.operation.name": "xread", + "lineageweave.stream.kind": "post_content", + }, + ): + raise + if not batches: + return last_id + with traced( + "lineageweave.valkey.post_content_batch", + { + "db.system": "redis", + "db.operation.name": "xread", + "lineageweave.stream.kind": "post_content", + }, + ): + for _stream_name, entries in batches: + for entry_id, fields in entries: + post_id = str(fields.get("post_id", "")).strip() + digest = str(fields.get("source_body_sha256", "")).strip() + try: + UUID(post_id) + except ValueError: + post_id = "" + if post_id and len(digest) == 64: + await process_post_content_job( + pool, + post_id=post_id, + source_body_digest=digest, + vision_factory=vision_factory, + embedding_factory=embedding_factory, + structure_factory=structure_factory, + ) + last_id = str(entry_id) return last_id @@ -348,11 +388,17 @@ async def run_post_content_worker( if now - last_recovery >= _RECOVERY_INTERVAL_SECONDS: await republish_queued_post_content_jobs(client, pool) last_recovery = now - last_id = await consume_post_content_stream_once( - client, - pool, - last_id=last_id, - vision_factory=vision_factory, - embedding_factory=embedding_factory, - structure_factory=structure_factory, - ) + try: + last_id = await consume_post_content_stream_once( + client, + pool, + last_id=last_id, + vision_factory=vision_factory, + embedding_factory=embedding_factory, + structure_factory=structure_factory, + ) + except (redis.RedisError, OSError) as exc: + _logger.warning( + "post-content Valkey poll failed; retrying (error_type=%s)", type(exc).__name__ + ) + await asyncio.sleep(_BROKER_RECOVERY_DELAY_SECONDS) diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 492b82175..79d4826cb 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -3948,6 +3948,11 @@ def test_evaluate_is_unavailable_without_orchestrator(client, demo_analyst_token headers={"Authorization": f"Bearer {demo_analyst_token}"}, ) assert response.status_code == 503 + assert response.json()["detail"] == ( + "Post evaluation is unavailable. Ask an administrator to configure the analysis service, " + "then retry." + ) + assert "ORCHESTRATOR_" not in response.text @pytest.mark.skipif( diff --git a/docker-compose.yml b/docker-compose.yml index 3d0db99b8..ea2f963a8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -117,6 +117,10 @@ services: # explicit bounded 8 MiB limit rather than an unbounded request size. CONTEXTUAL_ORCHESTRATOR_MAX_BODY_BYTES: ${CONTEXTUAL_ORCHESTRATOR_MAX_BODY_BYTES:-8388608} CONTEXTUAL_ORCHESTRATOR_ALLOWED_PROVIDER_HOSTS: ${CONTEXTUAL_ORCHESTRATOR_ALLOWED_PROVIDER_HOSTS:-host.docker.internal} + OTEL_SERVICE_NAME: ${OTEL_ORCHESTRATOR_SERVICE_NAME:-contextual-orchestrator} + # Do not set OTEL_EXPORTER_OTLP_ENDPOINT here. An empty + # ${OTEL_EXPORTER_OTLP_ENDPOINT:-} interpolation would wipe a value from + # env_file (${HOME}/.env). Export stays opt-in from that file or the host. command: ["python", "/app/start.py"] ports: - "${ORCHESTRATOR_PORT:-18000}:8000" @@ -161,6 +165,8 @@ services: OIDC_CLOCK_SKEW_SECONDS: ${OIDC_CLOCK_SKEW_SECONDS:-5} FRONTEND_ORIGINS: http://localhost:${FRONTEND_PORT:-15173} VALKEY_URL: redis://valkey:6379/0 + OTEL_SERVICE_NAME: ${OTEL_SERVICE_NAME:-lineageweave} + OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-} # Empty by default: every LLM/vision channel stays the Null client # (dropped, not faked). Set these to a running contextual-orchestrator # to turn the channels on. Provider credentials use LLM_GATEWAY_API_URL / diff --git a/docker/contextual-orchestrator/Dockerfile b/docker/contextual-orchestrator/Dockerfile index f4ef8eab7..0af60f58c 100644 --- a/docker/contextual-orchestrator/Dockerfile +++ b/docker/contextual-orchestrator/Dockerfile @@ -5,12 +5,16 @@ WORKDIR /app # Reuse the upstream implementation without copying it into LineageWeave. # Pin the runtime to a reviewed immutable upstream commit; model selection, # structured synthesis, and reasoning policy stay in contextual-orchestrator. -ADD https://github.com/ContextualWisdomLab/contextual-orchestrator/archive/7df051ac2b929e5910071ac1848d0447c5d6744e.tar.gz /tmp/contextual-orchestrator.tar.gz +ADD https://github.com/ContextualWisdomLab/contextual-orchestrator/archive/1a40e0f7ad10d1a24137d69d20e44fc9a5dcdd89.tar.gz /tmp/contextual-orchestrator.tar.gz RUN mkdir /tmp/contextual-orchestrator \ && tar -xzf /tmp/contextual-orchestrator.tar.gz --strip-components=1 -C /tmp/contextual-orchestrator \ && cp -R /tmp/contextual-orchestrator/contextual_orchestrator /app/contextual_orchestrator \ && cp -R /tmp/contextual-orchestrator/examples /app/examples \ && rm -rf /tmp/contextual-orchestrator /tmp/contextual-orchestrator.tar.gz \ + && python -m pip install --no-cache-dir \ + 'opentelemetry-api>=1.30.0' \ + 'opentelemetry-sdk>=1.30.0' \ + 'opentelemetry-exporter-otlp-proto-http>=1.30.0' \ && useradd --uid 10001 --no-create-home orchestrator COPY agents.json /app/agents.json diff --git a/docs/adr/0083-orchestrator-runtime-commit-pin.md b/docs/adr/0083-orchestrator-runtime-commit-pin.md index e5ea7083e..9bb6bd2b3 100644 --- a/docs/adr/0083-orchestrator-runtime-commit-pin.md +++ b/docs/adr/0083-orchestrator-runtime-commit-pin.md @@ -15,7 +15,7 @@ multi-agent. ## Decision `docker/contextual-orchestrator/Dockerfile` pins the downloaded archive to -commit `7df051ac2b929e5910071ac1848d0447c5d6744e`. The pin remains explicit +commit `1a40e0f7ad10d1a24137d69d20e44fc9a5dcdd89`. The pin remains explicit and immutable until the reviewed upstream change is superseded; it is not a moving `main` reference and it is not a LineageWeave monkey patch. diff --git a/docs/adr/0122-otel-session-observability.md b/docs/adr/0122-otel-session-observability.md new file mode 100644 index 000000000..01865559d --- /dev/null +++ b/docs/adr/0122-otel-session-observability.md @@ -0,0 +1,80 @@ +# ADR 0122: Correlate product, orchestrator, and Valkey telemetry by post session + +## Status + +Accepted. + +## Context + +Post structure, VISION, OCR, embeddings, summaries, and queue work can run in +different processes. Existing ADR 0071 already defines a deterministic, +post-scoped session metadata value, but transport and queue failures were not +visible as one trace. The proposed organization control contract currently +lives in [GRC PR #51 at exact head +`1a8f90dd15f37ffc86b8a0efd217a8b2812e5f99`](https://github.com/ContextualWisdomLab/governance-risk-compliance/blob/1a8f90dd15f37ffc86b8a0efd217a8b2812e5f99/docs/adr/0009-opentelemetry-request-telemetry.md), +not on protected `develop`. Until that proposal lands, this accepted +LineageWeave ADR remains the operative product boundary and the GRC proposal +must not be cited as protected organization evidence. + +## Decision + +1. LineageWeave uses the OpenTelemetry Python API and SDK and exports OTLP only + when OTEL_EXPORTER_OTLP_ENDPOINT is explicitly configured. The exporter + treats that value as a base URL and sends traces, metrics, and correlated + logs to the normalized /v1/traces, /v1/metrics, and /v1/logs signal + endpoints. The service resource name is lineageweave unless the operator + overrides it with the standard OTEL_SERVICE_NAME variable. A blank or unset + endpoint leaves the SDK unconfigured so a later operator value can still + enable export. +2. Every contextual-orchestrator POST carries the existing + `lineageweave_post_session_id` as `X-LineageWeave-Session-Id`. The + orchestrator binds it to the request context and adds it to provider spans, + so chat, Responses, structured output, VISION, and embedding work for one + post can be investigated together. +3. LineageWeave emits bounded HTTP and Valkey operation spans. HTTP client + failures follow the OpenTelemetry HTTP semantic conventions: error + responses and invalid response bodies end the client span with an error. + Valkey spans identify the operation and logical stream kind, not the stream + key, post body, summary, actor, source identifiers, token, or provider + response. Idle blocking reads do not emit empty spans; a non-empty batch + emits one bounded consumption span. +4. Failure telemetry uses two fixed outcomes: `provider_unavailable` for + explicitly classified provider, transport, or schema failures, and + `internal_error` for unexpected programming failures. The counter labels + contain only operation code and outcome, so high-cardinality session IDs and + exception classes remain in bounded structured logs and traces instead of + metric labels. +5. Failure logs contain operation, error type, the bounded session + correlation, and the active W3C TraceId and SpanId so another agent can + join the structured log to the Error span. Unexpected failures may include + a bounded stack trace, but never the exception value, prompt, response, + source body, credential, actor, or tenant identifier. They do not become a + second evidence database. GRC may consume aggregate control evidence and + OTLP-derived SLO signals through its existing contracts; LineageWeave does + not copy GRC tables or credentials. +6. No ad hoc session table is introduced. The existing normalized post-scoped + session metadata remains the source of correlation. + +## Consequences + +Operators can follow a slow or failed post-content job from the LineageWeave +HTTP client through contextual-orchestrator and Valkey without exposing source +content. Global Ask and post chat return a reader-safe generic 503 while GRC +can distinguish provider unavailability from an internal defect. An OTLP +collector is a deployment concern, not a local default, so a developer stack +remains usable without a telemetry backend. Raw session IDs remain +correlation data and must not be used as tenant, actor, or evidence labels in +GRC dashboards. + +## References + +OpenTelemetry Authors. (n.d.). *Manual instrumentation with OpenTelemetry +Python*. Retrieved August 21, 2026, from +https://opentelemetry.io/docs/languages/python/instrumentation/ + +OpenTelemetry Authors. (n.d.). *Service semantic conventions*. Retrieved +August 21, 2026, from https://opentelemetry.io/docs/specs/semconv/registry/attributes/service/ + +OpenTelemetry Authors. (n.d.). *Semantic conventions for HTTP spans*. +Retrieved August 22, 2026, from +https://opentelemetry.io/docs/specs/semconv/http/http-spans/ diff --git a/docs/doctoring/OPENTELEMETRY_REFERENCES.md b/docs/doctoring/OPENTELEMETRY_REFERENCES.md new file mode 100644 index 000000000..b69547009 --- /dev/null +++ b/docs/doctoring/OPENTELEMETRY_REFERENCES.md @@ -0,0 +1,51 @@ +# OpenTelemetry references and implementation traceability + +## Normative and alignment references + +- OpenTelemetry Authors. (n.d.). *Manual instrumentation with OpenTelemetry + Python*. Retrieved August 21, 2026, from + https://opentelemetry.io/docs/languages/python/instrumentation/ +- OpenTelemetry Authors. (n.d.). *Service semantic conventions*. Retrieved + August 21, 2026, from + https://opentelemetry.io/docs/specs/semconv/registry/attributes/service/ +- Organization alignment proposal, not protected-`develop` evidence: + ContextualWisdomLab governance-risk-compliance PR #51 exact head + `1a8f90dd15f37ffc86b8a0efd217a8b2812e5f99`, *ADR 0009: Emit bounded + OpenTelemetry request telemetry*. Retrieved August 23, 2026, from + https://github.com/ContextualWisdomLab/governance-risk-compliance/blob/1a8f90dd15f37ffc86b8a0efd217a8b2812e5f99/docs/adr/0009-opentelemetry-request-telemetry.md + +## Implementation mapping + +| Concern | Implementation | Evidence boundary | +| --- | --- | --- | +| Service resource | `OTEL_SERVICE_NAME`, default `lineageweave` | One logical service name per deployment | +| Post correlation | Existing ADR 0071 session metadata plus `X-LineageWeave-Session-Id` | Correlation only; not identity or authorization | +| LLM/VISION/embedding transport | `lineageweave.http_client.post_json` | Method, peer, bounded path, status; no body or credential | +| Valkey queue | `backend/app/*worker.py` and stream producers | Operation and logical stream kind; no stream key or event content | +| Server failure metric | `lineageweave.server.failures` | Fixed operation/outcome labels; no session or exception labels | +| Server failure log/trace | `record_server_failure` | Error class, bounded stack, TraceId, and SpanId; no exception value or source content | +| Export | OTEL_EXPORTER_OTLP_ENDPOINT | Disabled by default; base URL normalized to /v1/traces, /v1/metrics, and /v1/logs | + +## Correlation fields other agents must read + +Join one failed Ask or post-chat operation across API, orchestrator, and +Valkey by these bounded fields. Do not treat `session_id` as a tenant, actor, +or evidence identifier. + +| Field | Where | Shape | +| --- | --- | --- | +| `trace_id` | structured log extra and span context | 32-character lowercase hex W3C TraceId | +| `span_id` | structured log extra and span context | 16-character lowercase hex W3C SpanId | +| `traceparent` | outbound HTTP to contextual-orchestrator/TEPP | W3C Trace Context header `00-{trace_id}-{span_id}-{flags}` | +| `operation_code` | log extra, span attribute `lineageweave.operation_code`, metric label | one of `global_ask`, `post_chat`, `http_post_json`, `http_get_json`, `post_content_ingestion`, `unknown` | +| `failure_outcome` | log extra, span attribute `lineageweave.failure_outcome`, metric label | `provider_unavailable` or `internal_error` | +| `error_type` | log extra and span attribute `lineageweave.error_type` | exception class name only | +| `session_id` | log extra, span attribute `lineageweave.session_id`, `X-LineageWeave-Session-Id` | bounded ADR 0071 post session; correlation only | + +Child Valkey spans created with `traced()` in the same process inherit the +parent TraceId. Same-process inheritance is the Valkey correlation contract; +stream payloads do not carry post bodies or raw stream keys. + +The GRC repository remains the organization control and evidence owner. This +repository emits operational signals and does not copy GRC tables or persist +provider credentials. diff --git a/lineageweave/caldav_client.py b/lineageweave/caldav_client.py index 793462fe0..3653cf791 100644 --- a/lineageweave/caldav_client.py +++ b/lineageweave/caldav_client.py @@ -44,7 +44,9 @@ def __init__(self, base_url: str) -> None: def list_events(self) -> list[CalDavEvent]: """Fetch and parse events from the configured CalDAV endpoint.""" - payload = get_json(self._events_url, timeout=10) + payload = get_json( + self._events_url, timeout=10, service_peer_name="caldav" + ) rows = payload.get("events") if not isinstance(rows, list): return [] diff --git a/lineageweave/embedding_client.py b/lineageweave/embedding_client.py index d76b12944..f6a6306bc 100644 --- a/lineageweave/embedding_client.py +++ b/lineageweave/embedding_client.py @@ -111,6 +111,7 @@ def embed_many(self, texts: list[str]) -> list[list[float]]: f"{self._base_url}/batch/embeddings/{batch_id}", headers=headers, timeout=self._timeout, + service_peer_name="contextual-orchestrator", ) vectors = self._vectors(response, len(texts)) diff --git a/lineageweave/http_client.py b/lineageweave/http_client.py index 0f4be13df..d1791cd05 100644 --- a/lineageweave/http_client.py +++ b/lineageweave/http_client.py @@ -15,16 +15,19 @@ import http.client import json import ssl +from collections.abc import Callable from urllib.parse import urlencode, urlparse import certifi from .llm_context import current_llm_metadata +from .observability import current_session_id, inject_trace_context, traced # Some interpreter distributions don't reliably inherit the OS trust store. # Pointing at certifi keeps full chain validation without weakening TLS. _SSL_CONTEXT = ssl.create_default_context(cafile=certifi.where()) _ALLOWED_SCHEMES = frozenset({"http", "https"}) +_SESSION_HEADER_PEERS = frozenset({"contextual-orchestrator", "tepp"}) class HttpClientError(RuntimeError): @@ -246,24 +249,49 @@ def post_json( *, headers: dict[str, str], timeout: float, + service_peer_name: str = "contextual-orchestrator", ) -> dict: """POST ``payload`` as JSON to ``url`` and return the decoded object. Raises: ValueError: ``url`` is not an ``http`` / ``https`` URL with a host. HttpClientError: the server responded with HTTP >= 400 or non-JSON. + + ``service_peer_name`` is a bounded service name used for the request span. """ - status, raw = _request( - "POST", - url, - body=json_request_body(payload), - headers={"content-type": "application/json", **headers}, - timeout=timeout, - ) hostname = urlparse(url).hostname or url - if status >= 400: - raise HttpClientError(f"HTTP {status} from {hostname}") - return _decode_json_object(raw, hostname) + request_headers = {"content-type": "application/json", **headers} + session_id = current_session_id() + if session_id: + request_headers["x-lineageweave-session-id"] = session_id + with traced( + "lineageweave.http.post_json", + { + "http.request.method": "POST", + "lineageweave.operation_code": "http_post_json", + "service.peer.name": service_peer_name, + }, + ) as span: + inject_trace_context(request_headers) + status, raw = _request( + "POST", + url, + body=json_request_body(payload), + headers=request_headers, + timeout=timeout, + ) + if span is not None: + span.set_attribute("http.response.status_code", status) + if status >= 400: + if span is not None: + span.set_attribute("error.type", str(status)) + raise HttpClientError(f"HTTP {status} from {hostname}") + try: + return _decode_json_object(raw, hostname) + except HttpClientError: + if span is not None: + span.set_attribute("error.type", "HttpClientError") + raise def post_form( @@ -295,11 +323,62 @@ def post_form( return _decode_json_object(raw, hostname) +def _traced_get_json( + url: str, + *, + headers: dict[str, str] | None, + timeout: float, + decoder: Callable[[bytes, str], dict | list], + span_name: str, + service_peer_name: str, + maximum_response_bytes: int | None = None, + expected_response_media_type: str | None = None, +): + """GET ``url`` under one HTTP span and inject the active W3C context.""" + hostname = urlparse(url).hostname or url + request_headers = dict(headers or {}) + if service_peer_name in _SESSION_HEADER_PEERS: + session_id = current_session_id() + if session_id: + request_headers["x-lineageweave-session-id"] = session_id + with traced( + span_name, + { + "http.request.method": "GET", + "lineageweave.operation_code": "http_get_json", + "service.peer.name": service_peer_name, + }, + ) as span: + inject_trace_context(request_headers) + status, raw = _request( + "GET", + url, + body=None, + headers=request_headers, + timeout=timeout, + maximum_response_bytes=maximum_response_bytes, + expected_response_media_type=expected_response_media_type, + ) + if span is not None: + span.set_attribute("http.response.status_code", status) + if status >= 400: + if span is not None: + span.set_attribute("error.type", str(status)) + raise HttpClientError(f"HTTP {status} from {hostname}") + try: + return decoder(raw, hostname) + except HttpClientError: + if span is not None: + span.set_attribute("error.type", "HttpClientError") + raise + + def get_json( url: str, *, headers: dict[str, str] | None = None, timeout: float, + service_peer_name: str = "http-service", maximum_response_bytes: int | None = None, expected_response_media_type: str | None = None, ) -> dict: @@ -317,20 +396,16 @@ def get_json( HttpClientError: The response is too large, has the wrong media type, returns HTTP >= 400, or is not a JSON object. """ - - status, raw = _request( - "GET", + return _traced_get_json( url, - body=None, - headers=headers or {}, + headers=headers, timeout=timeout, + decoder=_decode_json_object, + span_name="lineageweave.http.get_json", + service_peer_name=service_peer_name, maximum_response_bytes=maximum_response_bytes, expected_response_media_type=expected_response_media_type, ) - hostname = urlparse(url).hostname or url - if status >= 400: - raise HttpClientError(f"HTTP {status} from {hostname}") - return _decode_json_object(raw, hostname) def get_json_list( @@ -338,6 +413,7 @@ def get_json_list( *, headers: dict[str, str] | None = None, timeout: float, + service_peer_name: str = "contextual-orchestrator", ) -> list: """GET ``url`` and return the decoded JSON array. @@ -348,15 +424,11 @@ def get_json_list( ValueError: ``url`` is not an ``http`` / ``https`` URL with a host. HttpClientError: the server responded with HTTP >= 400 or non-array JSON. """ - - status, raw = _request( - "GET", + return _traced_get_json( url, - body=None, - headers=headers or {}, + headers=headers, timeout=timeout, + decoder=_decode_json_list, + span_name="lineageweave.http.get_json_list", + service_peer_name=service_peer_name, ) - hostname = urlparse(url).hostname or url - if status >= 400: - raise HttpClientError(f"HTTP {status} from {hostname}") - return _decode_json_list(raw, hostname) diff --git a/lineageweave/observability.py b/lineageweave/observability.py new file mode 100644 index 000000000..9262a4135 --- /dev/null +++ b/lineageweave/observability.py @@ -0,0 +1,404 @@ +"""Bounded OpenTelemetry spans for product and infrastructure operations. + +The application emits useful correlation without placing post bodies, provider +credentials, actor identifiers, or arbitrary request paths in telemetry. +Export is opt-in through the standard OTLP environment variables. +""" + +from __future__ import annotations + +import logging +import os +import traceback +from collections.abc import Iterator, Mapping +from contextlib import contextmanager, nullcontext +from typing import Any + +try: + from opentelemetry import metrics, trace + from opentelemetry.propagate import inject as _otel_inject + from opentelemetry.trace import Status, StatusCode +except ImportError: # pragma: no cover - dependency is declared by the project + metrics = None # type: ignore[assignment] + trace = None # type: ignore[assignment] + _otel_inject = None + Status = None # type: ignore[assignment,misc] + StatusCode = None # type: ignore[assignment,misc] + +_LOGGER = logging.getLogger(__name__) +_CONFIGURED = False +_TRACER_NAME = "lineageweave" +_FAILURE_COUNTER: Any = None +_TRACE_PROVIDER: Any = None +_METER_PROVIDER: Any = None +_LOG_PROVIDER: Any = None +_LOG_HANDLER: Any = None +_SERVER_FAILURE_OUTCOMES = {"provider_unavailable", "internal_error"} +_ALLOWED_ATTRIBUTE_KEYS = frozenset( + { + "db.operation.name", + "db.system", + "http.request.method", + "http.response.status_code", + "lineageweave.error_type", + "lineageweave.failure_outcome", + "lineageweave.operation_code", + "lineageweave.session_id", + "lineageweave.stream.kind", + "service.peer.name", + } +) +_ALLOWED_OPERATION_CODES = frozenset( + { + "global_ask", + "http_get_json", + "http_post_json", + "post_chat", + "post_content_ingestion", + "unknown", + } +) + + +def _bounded_session_id(value: object) -> str | None: + """Return a short, printable session correlation value or ``None``.""" + if not isinstance(value, str): + return None + value = value.strip() + if not value or not value.isprintable(): + return None + return value[:128] + + +def _otlp_trace_endpoint(endpoint: str) -> str: + """Turn an OTLP base endpoint into the explicit HTTP traces endpoint.""" + return _otlp_signal_endpoint(endpoint, "traces") + + +def _otlp_signal_endpoint(endpoint: str, signal: str) -> str: + """Turn an OTLP base endpoint into one explicit HTTP signal endpoint.""" + normalized = endpoint.rstrip("/") + suffix = f"/v1/{signal}" + if normalized.casefold().endswith(suffix): + return normalized + return f"{normalized}{suffix}" + + +def _otlp_metric_endpoint(endpoint: str) -> str: + """Turn an OTLP base endpoint into the explicit HTTP metrics endpoint.""" + return _otlp_signal_endpoint(endpoint, "metrics") + + +def _otlp_log_endpoint(endpoint: str) -> str: + """Turn an OTLP base endpoint into the explicit HTTP logs endpoint.""" + return _otlp_signal_endpoint(endpoint, "logs") + + +def _current_trace_ids() -> tuple[str, str]: + """Return the active span's TraceId and SpanId as lowercase hex, or blanks.""" + getter = getattr(trace, "get_current_span", None) if trace is not None else None + if getter is None: + return "", "" + span = getter() + context_getter = getattr(span, "get_span_context", None) + if span is None or context_getter is None: + return "", "" + context = context_getter() + if context is None or not getattr(context, "is_valid", False): + return "", "" + return format(context.trace_id, "032x"), format(context.span_id, "016x") + + +def current_session_id() -> str | None: + """Return the current post-scoped session without exposing other metadata.""" + from .llm_context import current_llm_metadata + + metadata = current_llm_metadata() or {} + value = metadata.get("lineageweave_post_session_id") or metadata.get("session_id") + return _bounded_session_id(value) + + +def inject_trace_context(carrier: dict[str, str]) -> None: + """Inject the active W3C trace context without adding request content.""" + if _otel_inject is not None: + _otel_inject(carrier) + + +def _safe_attributes( + attributes: Mapping[str, Any] | None, +) -> dict[str, str | int | float | bool]: + """Keep telemetry attributes scalar, bounded, and explicitly non-content.""" + result: dict[str, str | int | float | bool] = {} + for key, value in (attributes or {}).items(): + if not isinstance(key, str) or key not in _ALLOWED_ATTRIBUTE_KEYS: + continue + if key == "lineageweave.session_id": + value = _bounded_session_id(value) + if value is None: + continue + if isinstance(value, (dict, list, tuple, set)): + continue + if isinstance(value, str): + result[key] = value[:256] + elif isinstance(value, (bool, int, float)): + result[key] = value + session_id = current_session_id() + if session_id: + result.setdefault("lineageweave.session_id", session_id) + return result + + +def configure_telemetry(service_name: str = "lineageweave") -> None: + """Configure OTLP traces, metrics, and correlated logs when enabled.""" + global _CONFIGURED, _TRACE_PROVIDER, _METER_PROVIDER, _LOG_PROVIDER, _LOG_HANDLER + if _CONFIGURED or os.getenv("OTEL_SDK_DISABLED", "").lower() == "true": + return + endpoint = os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "").strip() + if trace is None or not endpoint: + return + try: + from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( + OTLPSpanExporter, + ) + from opentelemetry.sdk.resources import Resource + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.sdk.trace.export import BatchSpanProcessor + except ImportError: # pragma: no cover - guarded by the runtime extra + _LOGGER.warning("OpenTelemetry trace SDK/exporter is unavailable") + return + + resource = Resource.create({ + "service.name": os.getenv("OTEL_SERVICE_NAME", service_name), + "service.namespace": "contextualwisdomlab", + }) + provider = TracerProvider(resource=resource) + provider.add_span_processor( + BatchSpanProcessor( + OTLPSpanExporter(endpoint=_otlp_trace_endpoint(endpoint)) + ) + ) + trace.set_tracer_provider(provider) + _TRACE_PROVIDER = provider + _CONFIGURED = True + if metrics is not None: + try: + from opentelemetry.exporter.otlp.proto.http.metric_exporter import ( + OTLPMetricExporter, + ) + from opentelemetry.sdk.metrics import MeterProvider + from opentelemetry.sdk.metrics.export import PeriodicExportingMetricReader + except ImportError: # pragma: no cover - guarded by the runtime extra + _LOGGER.warning("OpenTelemetry metric SDK/exporter is unavailable") + else: + meter_provider = MeterProvider( + resource=resource, + metric_readers=[ + PeriodicExportingMetricReader( + OTLPMetricExporter(endpoint=_otlp_metric_endpoint(endpoint)) + ) + ], + ) + metrics.set_meter_provider(meter_provider) + _METER_PROVIDER = meter_provider + try: + from opentelemetry._logs import set_logger_provider + from opentelemetry.exporter.otlp.proto.http._log_exporter import ( + OTLPLogExporter, + ) + from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler + from opentelemetry.sdk._logs.export import BatchLogRecordProcessor + except ImportError: # pragma: no cover - guarded by the runtime extra + _LOGGER.warning("OpenTelemetry log SDK/exporter is unavailable") + return + try: + log_provider = LoggerProvider(resource=resource) + log_provider.add_log_record_processor( + BatchLogRecordProcessor( + OTLPLogExporter(endpoint=_otlp_log_endpoint(endpoint)) + ) + ) + set_logger_provider(log_provider) + handler = LoggingHandler(level=logging.WARNING, logger_provider=log_provider) + _LOGGER.addHandler(handler) + _LOG_PROVIDER = log_provider + _LOG_HANDLER = handler + except Exception: # noqa: BLE001 - export must stay fail-open + _LOGGER.warning("OpenTelemetry log exporter is unavailable") + + +def shutdown_telemetry() -> None: + """Flush configured OTLP providers without masking application shutdown.""" + global _CONFIGURED, _TRACE_PROVIDER, _METER_PROVIDER, _LOG_PROVIDER + global _LOG_HANDLER, _FAILURE_COUNTER + if _LOG_HANDLER is not None: + _LOGGER.removeHandler(_LOG_HANDLER) + _LOG_HANDLER = None + for provider_name, provider in ( + ("trace", _TRACE_PROVIDER), + ("metric", _METER_PROVIDER), + ("log", _LOG_PROVIDER), + ): + if provider is None: + continue + try: + provider.shutdown() + except Exception: # noqa: BLE001 - telemetry must not mask shutdown + _LOGGER.warning( + "telemetry.provider_shutdown_failed provider=%s", + provider_name, + ) + _TRACE_PROVIDER = None + _METER_PROVIDER = None + _LOG_PROVIDER = None + _FAILURE_COUNTER = None + _CONFIGURED = False + + +def _failure_counter() -> Any: + """Return the OTel counter without making telemetry a request dependency.""" + global _FAILURE_COUNTER + if _FAILURE_COUNTER is None and metrics is not None: + _FAILURE_COUNTER = metrics.get_meter(_TRACER_NAME).create_counter( + "lineageweave.server.failures", + description="Server failures classified by operation and outcome", + ) + return _FAILURE_COUNTER + + +def _stack_trace_without_exception(exc: BaseException) -> str: + """Bound a stack trace while excluding the exception value/message.""" + if exc.__traceback__ is None: + return "" + return "".join(traceback.format_tb(exc.__traceback__))[:4096] + + +def _annotate_failure_span( + span: Any, + operation_code: str, + outcome: str, + error_type: str, + stack_trace: str, +) -> None: + """Attach only bounded classification data to an active span.""" + safe = _safe_attributes( + { + "lineageweave.operation_code": operation_code, + "lineageweave.failure_outcome": outcome, + "lineageweave.error_type": error_type, + } + ) + for key, value in safe.items(): + span.set_attribute(key, value) + event_attributes: dict[str, str] = {"exception.type": error_type} + if stack_trace: + event_attributes["exception.stacktrace"] = stack_trace + span.add_event("exception", event_attributes) + if Status is not None and StatusCode is not None: + span.set_status(Status(StatusCode.ERROR)) + + +def record_server_failure( + operation_code: str, + exc: BaseException, + *, + outcome: str, +) -> None: + """Record a classified server failure without storing exception content. + + ``provider_unavailable`` and ``internal_error`` are the only metric + outcomes. The error class is retained in logs and spans, while the + exception value is intentionally never serialized. + """ + if outcome not in _SERVER_FAILURE_OUTCOMES: + raise ValueError(f"unsupported server failure outcome: {outcome}") + bounded_operation = operation_code.strip() if isinstance(operation_code, str) else "unknown" + if bounded_operation not in _ALLOWED_OPERATION_CODES: + bounded_operation = "unknown" + error_type = type(exc).__name__[:128] + session_id = current_session_id() or "" + counter = _failure_counter() + if counter is not None: + try: + counter.add( + 1, + { + "lineageweave.operation_code": bounded_operation, + "lineageweave.failure_outcome": outcome, + }, + ) + except Exception: # noqa: BLE001 # telemetry failure must not mask API failure + _LOGGER.warning("telemetry.metric_recording_failed") + + stack_trace = ( + _stack_trace_without_exception(exc) if outcome == "internal_error" else "" + ) + current = trace.get_current_span() if trace is not None else None + span_context: Any = nullcontext() + if current is not None and current.is_recording(): + _annotate_failure_span( + current, bounded_operation, outcome, error_type, stack_trace + ) + elif trace is not None: + span_context = trace.get_tracer(_TRACER_NAME).start_as_current_span( + "lineageweave.server.failure", + record_exception=False, + set_status_on_exception=False, + ) + + with span_context as span: + if span is not None: + _annotate_failure_span( + span, bounded_operation, outcome, error_type, stack_trace + ) + trace_id, span_id = _current_trace_ids() + _LOGGER.log( + logging.ERROR if outcome == "internal_error" else logging.WARNING, + "lineageweave.server_failure", + extra={ + "operation_code": bounded_operation, + "failure_outcome": outcome, + "error_type": error_type, + "session_id": session_id, + "stack_trace": stack_trace, + "trace_id": trace_id, + "span_id": span_id, + }, + ) + + +@contextmanager +def traced( + name: str, + attributes: Mapping[str, Any] | None = None, +) -> Iterator[Any]: + """Create a span and a prompt-safe log event for one bounded operation.""" + if trace is None: # pragma: no cover - dependency is declared by the project + yield None + return + tracer = trace.get_tracer(_TRACER_NAME) + with tracer.start_as_current_span( + name, record_exception=False, set_status_on_exception=False + ) as span: + safe = _safe_attributes(attributes) + for key, value in safe.items(): + span.set_attribute(key, value) + try: + yield span + except Exception as exc: + if Status is not None and StatusCode is not None: + span.add_event( + "exception", + {"exception.type": type(exc).__name__[:128]}, + ) + span.set_status(Status(StatusCode.ERROR)) + trace_id, span_id = _current_trace_ids() + _LOGGER.warning( + "telemetry.operation_failed operation=%s error_type=%s " + "session_id=%s trace_id=%s span_id=%s", + name, + type(exc).__name__, + safe.get("lineageweave.session_id", ""), + trace_id, + span_id, + ) + raise diff --git a/lineageweave/relation_verification.py b/lineageweave/relation_verification.py index 509c7552a..91ce8e113 100644 --- a/lineageweave/relation_verification.py +++ b/lineageweave/relation_verification.py @@ -144,6 +144,7 @@ def verify(self, organization_name: str, relationship_label: str) -> RelationVer body = get_json( f"{self._base_url}/search?q={quote(query, safe='')}&format=json", timeout=self._timeout, + service_peer_name="searxng", ) results = body.get("results") if not isinstance(results, list): diff --git a/pyproject.toml b/pyproject.toml index ba2d25af9..da95c7fcc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,9 @@ dependencies = [ # docs/ontology/lineageweave-kg.ttl and the standards-complete PROV-O # support profile (ADR 0011). Pure Python, no Rust/C toolchain. "rdflib>=7.0.0", + "opentelemetry-api>=1.30.0", + "opentelemetry-sdk>=1.30.0", + "opentelemetry-exporter-otlp-proto-http>=1.30.0", ] [build-system] diff --git a/scripts/estimate_llm_channel_weights.py b/scripts/estimate_llm_channel_weights.py index 70a35de9d..1613ceb90 100644 --- a/scripts/estimate_llm_channel_weights.py +++ b/scripts/estimate_llm_channel_weights.py @@ -274,6 +274,7 @@ async def _collect(args: argparse.Namespace) -> dict[str, object]: f"{base_url}/api/v1/batch_routing_jobs/{run['batch_job_id']}", headers={"authorization": f"Bearer {api_key}"}, timeout=_BATCH_TIMEOUT_SECONDS, + service_peer_name="contextual-orchestrator", ) if not _is_complete(polled): return { diff --git a/scripts/seed_demo_data.py b/scripts/seed_demo_data.py index 931840704..60b60fb54 100644 --- a/scripts/seed_demo_data.py +++ b/scripts/seed_demo_data.py @@ -91,6 +91,7 @@ def _fetch_demo_user_subjects(base_url: str, admin_user: str, admin_password: st f"{base_url}/admin/realms/{REALM}/users?{query}", headers={"Authorization": f"Bearer {admin_token}"}, timeout=10, + service_peer_name="oidc", ) if not users: raise SystemExit(f"Keycloak user '{username}' not found in realm '{REALM}' -- did the realm import run?") diff --git a/scripts/smoke_test_oidc.py b/scripts/smoke_test_oidc.py index a3d33d2a7..d98e1f876 100644 --- a/scripts/smoke_test_oidc.py +++ b/scripts/smoke_test_oidc.py @@ -42,7 +42,7 @@ def _wait_for_realm(issuer: str) -> None: last_error: Exception | None = None for _ in range(POLL_ATTEMPTS): try: - get_json(discovery_url, timeout=5) + get_json(discovery_url, timeout=5, service_peer_name="oidc") return except (HttpClientError, OSError, ValueError) as exc: last_error = exc @@ -84,7 +84,7 @@ def run(base_url: str) -> int: access_token = token_response["access_token"] print(f"Fetching live JWKS from {jwks_uri} and verifying the token's RS256 signature...") - jwks = get_json(jwks_uri, timeout=10) + jwks = get_json(jwks_uri, timeout=10, service_peer_name="oidc") signing_key = _signing_key_from_jwks(jwks, access_token) claims = jwt.decode( access_token, diff --git a/tests/test_activity_stream.py b/tests/test_activity_stream.py index 16bcedb52..40ea78f4f 100644 --- a/tests/test_activity_stream.py +++ b/tests/test_activity_stream.py @@ -12,6 +12,8 @@ ticket_created_summary, ticket_status_changed_summary, ) +from lineageweave.observability import traced +from tests.test_observability import attach_inmemory_tracer class _FakeStream: @@ -63,3 +65,35 @@ def test_publish_activity_event_sync_skips_a_matching_summary() -> None: assert len(client.entries) == 1 assert client.entries[0][1]["event_type"] == "ticket_created" assert "Send Northridge Grid the revised quote" in client.entries[0][1]["summary"] + + +def test_valkey_child_span_shares_parent_trace_id(monkeypatch) -> None: + """Same-process Valkey work inherits the parent TraceId.""" + from opentelemetry import trace + + attach_inmemory_tracer(monkeypatch) + captured: dict[str, str] = {} + + class _Client(_FakeStream): + def xadd(self, key: str, fields: dict[str, str], maxlen=None, approximate=None): + span = trace.get_current_span() + captured["trace_id"] = format(span.get_span_context().trace_id, "032x") + captured["span_id"] = format(span.get_span_context().span_id, "016x") + return super().xadd(key, fields, maxlen=maxlen, approximate=approximate) + + with traced("lineageweave.test.parent"): + parent_context = trace.get_current_span().get_span_context() + parent_trace_id = format(parent_context.trace_id, "032x") + parent_span_id = format(parent_context.span_id, "016x") + publish_activity_event_sync( + _Client(), + "post-1", + "ticket_created", + "acct-1", + ticket_created_summary("Send Northridge Grid the revised quote"), + ) + + assert parent_trace_id != "0" * 32 + assert captured["trace_id"] == parent_trace_id + assert captured["span_id"] != "0" * 16 + assert captured["span_id"] != parent_span_id diff --git a/tests/test_analysis_run_worker.py b/tests/test_analysis_run_worker.py index e492b5a2e..847f72e3b 100644 --- a/tests/test_analysis_run_worker.py +++ b/tests/test_analysis_run_worker.py @@ -2,9 +2,14 @@ from __future__ import annotations +import asyncio +from contextlib import contextmanager +from unittest.mock import Mock + import pytest +import redis.asyncio as redis -from backend.app import analysis_run_worker +from backend.app import analysis_run_worker, post_content_worker from backend.app.analysis_run_start import AnalysisRunStartError from lineageweave.adjudication_client import NullAdjudicationClient from lineageweave.tepp_client import TeppClient @@ -56,6 +61,181 @@ async def xread(self, _streams, *, count, block): ] +class _IdleValkey: + async def xread(self, _streams, *, count, block): + assert (count, block) == (10, 1000) + return [] + + +@pytest.mark.anyio +async def test_idle_worker_reads_do_not_emit_empty_spans(monkeypatch): + """Blocking poll timeouts stay silent until a real batch arrives.""" + analysis_trace = Mock() + post_content_trace = Mock() + monkeypatch.setattr(analysis_run_worker, "traced", analysis_trace) + monkeypatch.setattr(post_content_worker, "traced", post_content_trace) + + assert await analysis_run_worker.consume_analysis_run_stream_once( + _IdleValkey(), + _Pool(), + last_id="0-0", + tepp_client=TeppClient(), + adjudication_client=NullAdjudicationClient(), + ) == "0-0" + assert await post_content_worker.consume_post_content_stream_once( + _IdleValkey(), + _Pool(), + last_id="0-0", + vision_factory=lambda: None, + embedding_factory=lambda: None, + structure_factory=lambda: None, + ) == "0-0" + analysis_trace.assert_not_called() + post_content_trace.assert_not_called() + + +@pytest.mark.anyio +async def test_xread_failures_emit_diagnostic_spans_but_preserve_errors(monkeypatch): + """Broker failures are traced without turning idle polls into spans.""" + analysis_trace_names = [] + post_content_trace_names = [] + + @contextmanager + def analysis_trace(name, _attributes): + analysis_trace_names.append(name) + yield None + + @contextmanager + def post_content_trace(name, _attributes): + post_content_trace_names.append(name) + yield None + + class FailingValkey: + async def xread(self, _streams, *, count, block): + assert (count, block) == (10, 1000) + raise RuntimeError("synthetic broker outage") + + monkeypatch.setattr(analysis_run_worker, "traced", analysis_trace) + monkeypatch.setattr(post_content_worker, "traced", post_content_trace) + + with pytest.raises(RuntimeError, match="synthetic broker outage"): + await analysis_run_worker.consume_analysis_run_stream_once( + FailingValkey(), + _Pool(), + last_id="0-0", + tepp_client=TeppClient(), + adjudication_client=NullAdjudicationClient(), + ) + with pytest.raises(RuntimeError, match="synthetic broker outage"): + await post_content_worker.consume_post_content_stream_once( + FailingValkey(), + _Pool(), + last_id="0-0", + vision_factory=lambda: None, + embedding_factory=lambda: None, + structure_factory=lambda: None, + ) + + assert analysis_trace_names == ["lineageweave.valkey.analysis_outbox_xread"] + assert post_content_trace_names == ["lineageweave.valkey.post_content_xread"] + + +@pytest.mark.anyio +async def test_workers_retry_transient_broker_errors_without_dropping_the_task(monkeypatch): + """A transient Valkey outage is retried; cancellation still stops each worker.""" + analysis_calls = 0 + post_content_calls = 0 + + class RecoveringAnalysisValkey: + async def xread(self, _streams, *, count, block): + nonlocal analysis_calls + assert (count, block) == (10, 1000) + analysis_calls += 1 + if analysis_calls == 1: + raise redis.RedisError("synthetic broker outage") + raise asyncio.CancelledError + + class RecoveringPostContentValkey: + async def xrevrange(self, _stream, *, count): + assert count == 1 + return [] + + async def xread(self, _streams, *, count, block): + nonlocal post_content_calls + assert (count, block) == (10, 1000) + post_content_calls += 1 + if post_content_calls == 1: + raise redis.RedisError("synthetic broker outage") + raise asyncio.CancelledError + + async def no_sleep(*_args): + return None + + monkeypatch.setattr(analysis_run_worker.asyncio, "sleep", no_sleep) + monkeypatch.setattr(post_content_worker.asyncio, "sleep", no_sleep) + monkeypatch.setattr(post_content_worker, "republish_queued_post_content_jobs", no_sleep) + + with pytest.raises(asyncio.CancelledError): + await analysis_run_worker.run_analysis_run_worker( + RecoveringAnalysisValkey(), + _Pool(), + tepp_client=TeppClient(), + adjudication_client=NullAdjudicationClient(), + ) + with pytest.raises(asyncio.CancelledError): + await post_content_worker.run_post_content_worker( + RecoveringPostContentValkey(), + _Pool(), + vision_factory=lambda: None, + embedding_factory=lambda: None, + structure_factory=lambda: None, + ) + + assert analysis_calls == 2 + assert post_content_calls == 2 + + +@pytest.mark.anyio +async def test_post_content_batch_advances_past_a_malformed_event(monkeypatch): + """A real batch is traced and malformed wake-up data stays untrusted.""" + calls = [] + + async def fake_process(_pool, **kwargs): + calls.append(kwargs) + + monkeypatch.setattr(post_content_worker, "process_post_content_job", fake_process) + + class MalformedValkey: + async def xread(self, _streams, *, count, block): + assert (count, block) == (10, 1000) + return [ + ( + "post-content", + [ + ("1-0", {"post_id": "not-a-uuid"}), + ( + "1-1", + { + "post_id": "00000000-0000-0000-0000-000000000001", + "source_body_sha256": "a" * 64, + }, + ), + ], + ) + ] + + assert await post_content_worker.consume_post_content_stream_once( + MalformedValkey(), + _Pool(), + last_id="0-0", + vision_factory=lambda: None, + embedding_factory=lambda: None, + structure_factory=lambda: None, + ) == "1-1" + assert calls[0]["post_id"] == "00000000-0000-0000-0000-000000000001" + assert calls[0]["source_body_digest"] == "a" * 64 + + @pytest.mark.anyio async def test_consumer_forwards_valid_event_and_skips_malformed_event(monkeypatch): calls = [] diff --git a/tests/test_caldav_client.py b/tests/test_caldav_client.py index 2cacb3539..f6453231d 100644 --- a/tests/test_caldav_client.py +++ b/tests/test_caldav_client.py @@ -21,8 +21,8 @@ def test_missing_base_url_drops_only_the_optional_caldav_channel() -> None: def test_http_client_reads_valid_events_and_ignores_malformed_rows(monkeypatch) -> None: received = {} - def fake_get_json(url: str, *, timeout: float) -> dict: - received.update(url=url, timeout=timeout) + def fake_get_json(url: str, *, timeout: float, **kwargs) -> dict: + received.update(url=url, timeout=timeout, **kwargs) return { "events": [ { @@ -46,7 +46,11 @@ def fake_get_json(url: str, *, timeout: float) -> dict: assert client.list_events() == [ CalDavEvent("event-1", "Review", "2026-08-19T09:00:00Z") ] - assert received == {"url": "https://calendar.example/caldav/events", "timeout": 10} + assert received == { + "url": "https://calendar.example/caldav/events", + "timeout": 10, + "service_peer_name": "caldav", + } def test_invalid_caldav_url_is_rejected() -> None: diff --git a/tests/test_embedding_client.py b/tests/test_embedding_client.py index 0ab4d0f8d..b3090c964 100644 --- a/tests/test_embedding_client.py +++ b/tests/test_embedding_client.py @@ -96,7 +96,8 @@ def fake_post_json(url, payload, *, headers, timeout): calls.append(("post", url, payload, headers)) return {"batch_id": "synthetic-batch", "status": "queued"} - def fake_get_json(url, *, headers, timeout): + def fake_get_json(url, *, headers, timeout, service_peer_name): + assert service_peer_name == "contextual-orchestrator" calls.append(("get", url, headers)) return { "batch_id": "synthetic-batch", diff --git a/tests/test_http_client.py b/tests/test_http_client.py index 5c53fc8cd..546bca797 100644 --- a/tests/test_http_client.py +++ b/tests/test_http_client.py @@ -10,9 +10,12 @@ HttpClientError, chat_completion_content, get_json, + get_json_list, post_form, post_json, ) +from lineageweave.observability import traced +from tests.test_observability import attach_inmemory_tracer @pytest.mark.parametrize( @@ -43,6 +46,8 @@ def do_GET(self) -> None: # noqa: N802 -- BaseHTTPRequestHandler API "path": self.path, "method": "GET", "authorization": self.headers.get("authorization"), + "traceparent": self.headers.get("traceparent"), + "session": self.headers.get("x-lineageweave-session-id"), } if "/users" in self.path: payload: object = [{"id": "sub-1", "username": "demo.analyst"}] @@ -63,6 +68,7 @@ def do_POST(self) -> None: # noqa: N802 -- BaseHTTPRequestHandler API "authorization": self.headers.get("authorization"), "content_type": self.headers.get("content-type"), "payload": raw.decode("utf-8"), + "traceparent": self.headers.get("traceparent"), } echo = raw.decode("utf-8") try: @@ -164,6 +170,92 @@ def test_get_json_fetches_json_from_http_endpoint() -> None: assert _JsonHandler.received["authorization"] == "Bearer test-token" +def _traceparent_trace_id(header: str | None) -> str: + assert header is not None + parts = header.split("-") + assert len(parts) == 4 + assert parts[0] == "00" + assert len(parts[1]) == 32 + assert len(parts[2]) == 16 + return parts[1] + + +def test_post_json_and_get_json_inject_parent_traceparent(monkeypatch) -> None: + """Shipped HTTP client injects the parent TraceId as W3C traceparent.""" + attach_inmemory_tracer(monkeypatch) + captured: dict[str, str | None] = {} + server, base = _serve(_JsonHandler) + try: + with traced("lineageweave.test.parent"): + from opentelemetry import trace + + parent_trace_id = format( + trace.get_current_span().get_span_context().trace_id, "032x" + ) + _JsonHandler.received = {} + post_json( + f"{base}/v1/chat/completions", + {}, + headers={}, + timeout=2.0, + service_peer_name="contextual-orchestrator", + ) + captured["post"] = _JsonHandler.received.get("traceparent") + _JsonHandler.received = {} + get_json( + f"{base}/v1/models", + timeout=2.0, + service_peer_name="tepp", + ) + captured["get"] = _JsonHandler.received.get("traceparent") + _JsonHandler.received = {} + get_json_list( + f"{base}/admin/users", + timeout=2.0, + service_peer_name="tepp", + ) + captured["list"] = _JsonHandler.received.get("traceparent") + finally: + server.shutdown() + + assert parent_trace_id != "0" * 32 + assert _traceparent_trace_id(captured["post"]) == parent_trace_id + assert _traceparent_trace_id(captured["get"]) == parent_trace_id + assert _traceparent_trace_id(captured["list"]) == parent_trace_id + + +def test_get_json_session_header_stays_on_orchestrator_peers(monkeypatch) -> None: + """Searxng/CalDAV/OIDC GETs keep W3C context without the post session header.""" + from lineageweave.llm_context import use_llm_metadata + + attach_inmemory_tracer(monkeypatch) + server, base = _serve(_JsonHandler) + try: + with use_llm_metadata({"lineageweave_post_session_id": "post-session-1"}): + _JsonHandler.received = {} + get_json(f"{base}/search", timeout=2.0, service_peer_name="searxng") + searxng = dict(_JsonHandler.received) + _JsonHandler.received = {} + get_json(f"{base}/generic", timeout=2.0) + generic = dict(_JsonHandler.received) + _JsonHandler.received = {} + get_json( + f"{base}/v1/models", + timeout=2.0, + service_peer_name="contextual-orchestrator", + ) + orchestrator = dict(_JsonHandler.received) + finally: + server.shutdown() + + assert searxng.get("session") is None + assert searxng.get("traceparent") + assert generic.get("session") is None + assert generic.get("traceparent") + assert orchestrator.get("session") == "post-session-1" + assert orchestrator.get("traceparent") + + @pytest.mark.parametrize("include_length", [True, False]) def test_get_json_rejects_responses_over_explicit_byte_limit( include_length: bool, diff --git a/tests/test_observability.py b/tests/test_observability.py new file mode 100644 index 000000000..16a29a2c6 --- /dev/null +++ b/tests/test_observability.py @@ -0,0 +1,345 @@ +"""Tests for prompt-safe session propagation and tracing boundaries.""" + +from __future__ import annotations + +import logging + +import pytest +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace import StatusCode + +from lineageweave import http_client, observability +from lineageweave.llm_context import use_llm_metadata +from lineageweave.observability import ( + _bounded_session_id, + _otlp_log_endpoint, + _otlp_trace_endpoint, + _safe_attributes, + current_session_id, + record_server_failure, + shutdown_telemetry, + traced, +) + + +def attach_inmemory_tracer(monkeypatch: pytest.MonkeyPatch) -> InMemorySpanExporter: + """Drive the shipped tracer through a real in-memory SDK exporter.""" + exporter = InMemorySpanExporter() + provider = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + monkeypatch.setattr(observability.trace, "get_tracer", provider.get_tracer) + return exporter + + +def test_post_json_sends_post_session_header(monkeypatch): + """One post session reaches the orchestrator as a transport header.""" + captured = {} + + def fake_request(method, url, *, body, headers, timeout): + captured.update(method=method, headers=headers) + return 200, b"{}" + + monkeypatch.setattr(http_client, "_request", fake_request) + attach_inmemory_tracer(monkeypatch) + with use_llm_metadata({"lineageweave_post_session_id": "post-session-1"}): + http_client.post_json( + "https://orchestrator.example/v1/chat/completions", + {}, + headers={}, + timeout=1, + ) + + assert captured["method"] == "POST" + assert captured["headers"]["x-lineageweave-session-id"] == "post-session-1" + assert captured["headers"]["traceparent"].startswith("00-") + _version, trace_id, span_id, _flags = captured["headers"]["traceparent"].split("-") + assert len(trace_id) == 32 + assert len(span_id) == 16 + assert trace_id != "0" * 32 + + +def test_post_json_marks_http_and_decode_failures_inside_the_span(monkeypatch): + """HTTP and invalid-body failures end the active client span as errors.""" + for status, raw, error_type in ( + (503, b"{}", "503"), + (200, b"not-json", "HttpClientError"), + ): + captured = {"attributes": {}} + + class _Span: + def set_attribute(self, key, value): + captured["attributes"][key] = value + + class _SpanContext: + def __enter__(self): + return _Span() + + def __exit__(self, exception_type, *_args): + captured["exception_type"] = exception_type + return False + + monkeypatch.setattr(http_client, "traced", lambda *_args, **_kwargs: _SpanContext()) + monkeypatch.setattr( + http_client, + "_request", + lambda *_args, **_kwargs: (status, raw), + ) + + try: + http_client.post_json( + "https://orchestrator.example/v1/chat/completions", + {}, + headers={}, + timeout=1, + ) + except http_client.HttpClientError: + pass + else: # pragma: no cover + raise AssertionError("invalid provider responses must fail closed") + + assert captured["exception_type"] is http_client.HttpClientError + assert captured["attributes"]["error.type"] == error_type + + +def test_current_session_id_reads_existing_context(): + """Telemetry reuses the existing normalized LLM context, not a new store.""" + with use_llm_metadata({"lineageweave_post_session_id": "post-session-2"}): + assert current_session_id() == "post-session-2" + + +def test_session_correlation_is_printable_and_bounded(): + """Session correlation rejects controls and caps values before export.""" + assert _bounded_session_id(" session-3 ") == "session-3" + assert _bounded_session_id("session\n3") is None + assert _bounded_session_id("session\x7f3") is None + assert _bounded_session_id("x" * 129) == "x" * 128 + assert _bounded_session_id(3) is None + + +def test_safe_attributes_drops_unknown_keys_and_invalid_session_values(): + """Telemetry keeps only the allowlisted scalar boundary attributes.""" + assert _safe_attributes( + { + "request.path": "/private/synthetic", + "lineageweave.session_id": "bad\nvalue", + "service.peer.name": "tepp", + "http.response.status_code": 503, + } + ) == { + "service.peer.name": "tepp", + "http.response.status_code": 503, + } + + +def test_traced_rethrows_provider_errors(): + """Observability never converts a failed provider operation into success.""" + try: + with traced("lineageweave.test.failure"): + raise RuntimeError("provider failure") + except RuntimeError as exc: + assert str(exc) == "provider failure" + else: # pragma: no cover + raise AssertionError("traced must preserve operation failures") + + +def test_traced_disables_automatic_exception_recording(monkeypatch): + """OTel must not serialize an exception value or implicit stack trace.""" + captured = {} + + class _Span: + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def set_attribute(self, _key, _value): + pass + + def add_event(self, name, attributes): + captured["event"] = (name, attributes) + + def set_status(self, _status): + pass + + class _Tracer: + def start_as_current_span(self, name, **kwargs): + captured["span"] = (name, kwargs) + return _Span() + + class _Trace: + def get_tracer(self, _name): + return _Tracer() + + monkeypatch.setattr(observability, "trace", _Trace()) + try: + with observability.traced("lineageweave.test.sensitive"): + raise RuntimeError("secret provider response") + except RuntimeError: + pass + + assert captured["span"] == ( + "lineageweave.test.sensitive", + {"record_exception": False, "set_status_on_exception": False}, + ) + assert captured["event"][1] == {"exception.type": "RuntimeError"} + + +def test_safe_attributes_keeps_allowlisted_operation_code(): + """Endpoint spans retain the bounded operation dimension.""" + assert _safe_attributes( + {"lineageweave.operation_code": "post_chat"} + ) == {"lineageweave.operation_code": "post_chat"} + + +def test_otlp_base_endpoint_gets_trace_signal_path(): + """A configured collector base URL receives the HTTP traces signal path.""" + assert _otlp_trace_endpoint("http://collector:4318") == "http://collector:4318/v1/traces" + assert _otlp_trace_endpoint("http://collector:4318/v1/traces/") == "http://collector:4318/v1/traces" + + +def test_otlp_base_endpoint_gets_log_signal_path(): + """A configured collector base URL receives the HTTP logs signal path.""" + assert _otlp_log_endpoint("http://collector:4318") == "http://collector:4318/v1/logs" + assert _otlp_log_endpoint("http://collector:4318/v1/logs") == "http://collector:4318/v1/logs" + + +def test_shutdown_telemetry_flushes_configured_providers(monkeypatch): + """Application shutdown flushes traces, metrics, and logs without a raw error.""" + calls = [] + + class _Provider: + def __init__(self, name): + self.name = name + + def shutdown(self): + calls.append(self.name) + + class _Handler: + pass + + monkeypatch.setattr(observability, "_TRACE_PROVIDER", _Provider("trace")) + monkeypatch.setattr(observability, "_METER_PROVIDER", _Provider("metric")) + monkeypatch.setattr(observability, "_LOG_PROVIDER", _Provider("log")) + monkeypatch.setattr(observability, "_LOG_HANDLER", _Handler()) + monkeypatch.setattr(observability, "_FAILURE_COUNTER", object()) + monkeypatch.setattr(observability, "_CONFIGURED", True) + + shutdown_telemetry() + + assert calls == ["trace", "metric", "log"] + assert observability._TRACE_PROVIDER is None + assert observability._METER_PROVIDER is None + assert observability._LOG_PROVIDER is None + assert observability._LOG_HANDLER is None + assert observability._FAILURE_COUNTER is None + assert observability._CONFIGURED is False + + +def test_record_server_failure_shares_trace_ids_with_active_span( + monkeypatch, caplog: pytest.LogCaptureFixture +) -> None: + """Classified failures annotate the API span and log the same TraceId/SpanId.""" + exporter = attach_inmemory_tracer(monkeypatch) + caplog.set_level(logging.WARNING, logger="lineageweave.observability") + sensitive = "secret prompt and source body must not escape" + + try: + with traced( + "lineageweave.api.global_ask", + {"lineageweave.operation_code": "global_ask"}, + ): + try: + raise ValueError(sensitive) + except ValueError as exc: + record_server_failure("global_ask", exc, outcome="provider_unavailable") + raise + except ValueError: + pass + + spans = exporter.get_finished_spans() + api_span = next( + span for span in spans if span.name == "lineageweave.api.global_ask" + ) + assert api_span.status.status_code == StatusCode.ERROR + assert api_span.attributes["lineageweave.operation_code"] == "global_ask" + assert api_span.attributes["lineageweave.failure_outcome"] == "provider_unavailable" + assert api_span.attributes["lineageweave.error_type"] == "ValueError" + assert not any(span.name == "lineageweave.server.failure" for span in spans) + + record = next( + item for item in caplog.records if item.msg == "lineageweave.server_failure" + ) + expected_trace = format(api_span.get_span_context().trace_id, "032x") + expected_span = format(api_span.get_span_context().span_id, "016x") + assert record.trace_id == expected_trace + assert record.span_id == expected_span + assert record.operation_code == "global_ask" + assert record.failure_outcome == "provider_unavailable" + assert sensitive not in caplog.text + assert sensitive not in str(api_span.attributes) + for event in api_span.events: + assert sensitive not in str(event.attributes) + + +def test_record_server_failure_distinguishes_internal_error_outcome( + monkeypatch, caplog: pytest.LogCaptureFixture +) -> None: + """Unexpected faults keep a stack without serializing the exception value.""" + exporter = attach_inmemory_tracer(monkeypatch) + caplog.set_level(logging.WARNING, logger="lineageweave.observability") + sensitive = "internal prompt-like value must not escape" + + try: + with traced( + "lineageweave.api.post_chat", + {"lineageweave.operation_code": "post_chat"}, + ): + try: + raise AttributeError(sensitive) + except AttributeError as exc: + record_server_failure("post_chat", exc, outcome="internal_error") + raise + except AttributeError: + pass + + spans = exporter.get_finished_spans() + api_span = next(span for span in spans if span.name == "lineageweave.api.post_chat") + record = next( + item for item in caplog.records if item.msg == "lineageweave.server_failure" + ) + assert record.failure_outcome == "internal_error" + assert record.stack_trace + assert record.trace_id == format(api_span.get_span_context().trace_id, "032x") + assert record.span_id == format(api_span.get_span_context().span_id, "016x") + assert api_span.attributes["lineageweave.failure_outcome"] == "internal_error" + assert sensitive not in record.stack_trace + assert sensitive not in caplog.text + + +def test_unknown_operation_code_maps_to_fallback( + monkeypatch, caplog: pytest.LogCaptureFixture +) -> None: + """Caller strings that are not allowlisted become the fixed unknown code.""" + attach_inmemory_tracer(monkeypatch) + caplog.set_level(logging.WARNING, logger="lineageweave.observability") + with traced("lineageweave.test.unknown"): + record_server_failure( + "not-an-allowed-code", RuntimeError("x"), outcome="internal_error" + ) + record = next( + item for item in caplog.records if item.msg == "lineageweave.server_failure" + ) + assert record.operation_code == "unknown" + + +def test_configure_telemetry_ignores_blank_endpoint(monkeypatch) -> None: + """Whitespace-only OTLP configuration must not latch a half-configured SDK.""" + monkeypatch.setattr(observability, "_CONFIGURED", False) + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", " ") + observability.configure_telemetry() + assert observability._CONFIGURED is False + assert observability._TRACE_PROVIDER is None + assert observability._LOG_PROVIDER is None diff --git a/tests/test_post_content_worker.py b/tests/test_post_content_worker.py index 52c2fb507..1cb073668 100644 --- a/tests/test_post_content_worker.py +++ b/tests/test_post_content_worker.py @@ -236,7 +236,8 @@ async def claim(*_args, **_kwargs): ), "empty-body skip must still emit a diagnostic log line" -def test_transient_provider_error_is_requeued_before_attempt_limit(monkeypatch) -> None: +def test_transient_provider_error_is_requeued_before_attempt_limit(monkeypatch, caplog) -> None: + caplog.set_level("WARNING", logger="lineageweave.observability") connection = _Connection(values=[2]) pool = _Pool(connection) @@ -272,8 +273,62 @@ async def persist(*_args, **_kwargs): ) updates = [args for query, args in connection.executed if "set status_code" in query] - assert any(args[1] == QUEUED and args[6] == "post_content_ingestion_failed" for args in updates) - assert "provider timeout" not in str(connection.executed) + assert any( + args[1] == QUEUED + and args[6] == "post_content_ingestion_failed" + and args[7] == post_content_worker._UNEXPECTED_FAILURE_DETAIL + for args in updates + ) + assert all("provider timeout" not in str(args) for args in updates) + assert "provider timeout" not in caplog.text + record = next( + item for item in caplog.records if item.msg == "lineageweave.server_failure" + ) + assert record.failure_outcome == "provider_unavailable" + + +def test_unexpected_worker_error_is_classified_as_internal(monkeypatch, caplog) -> None: + """Unexpected worker defects stay internal while their value remains private.""" + caplog.set_level("ERROR", logger="lineageweave.observability") + connection = _Connection(values=[2]) + pool = _Pool(connection) + + async def claim(*_args, **_kwargs): + return _row(RUNNING, 1) + + async def persist(*_args, **_kwargs): + raise TypeError("internal worker detail") + + monkeypatch.setattr(post_content_worker, "_claim_job", claim) + monkeypatch.setattr(post_content_worker, "persist_post_content", persist) + monkeypatch.setattr( + post_content_worker, + "load_settings", + lambda: SimpleNamespace( + embedding_model="embedding-model", + orchestrator_base_url="", + orchestrator_api_key="", + ), + ) + monkeypatch.setattr(post_content_worker, "normalize_post_body", lambda *_args: object()) + client = SimpleNamespace(available=True) + + asyncio.run( + post_content_worker.process_post_content_job( + pool, + post_id="00000000-0000-0000-0000-000000000001", + source_body_digest="a" * 64, + vision_factory=lambda: client, + embedding_factory=lambda: client, + structure_factory=lambda: client, + ) + ) + + record = next( + item for item in caplog.records if item.msg == "lineageweave.server_failure" + ) + assert record.failure_outcome == "internal_error" + assert "internal worker detail" not in caplog.text def test_failure_at_attempt_limit_is_terminal_and_visible() -> None: diff --git a/tests/test_server_diagnostics.py b/tests/test_server_diagnostics.py new file mode 100644 index 000000000..a8995540d --- /dev/null +++ b/tests/test_server_diagnostics.py @@ -0,0 +1,183 @@ +"""Reader-safe API failures and bounded OpenTelemetry diagnostics.""" + +from __future__ import annotations + +import asyncio +import logging +from types import SimpleNamespace +from typing import Self + +import pytest + +from backend.app import global_ask_queue, main +from lineageweave import observability +from tests.test_observability import attach_inmemory_tracer + + +class _Pool: + """Minimal async pool boundary for the endpoint unit tests.""" + + def acquire(self) -> Self: + return self + + async def __aenter__(self) -> Self: + return self + + async def __aexit__(self, *exc_info: object) -> None: + return None + + +class _FailingClient: + """Orchestrator-shaped client that raises one controlled exception.""" + + available = True + + def __init__(self, exc: BaseException) -> None: + self._exc = exc + + def answer(self, question: str, sources: object) -> object: + raise self._exc + + +def _call_ask(monkeypatch: pytest.MonkeyPatch, exc: BaseException) -> None: + async def _sources(*args: object, **kwargs: object) -> list[object]: + return [SimpleNamespace(post_id="synthetic-post-1")] + + monkeypatch.setattr(global_ask_queue, "gather_global_chat_sources", _sources) + with pytest.raises(main.HTTPException) as raised: + asyncio.run( + global_ask_queue.compute_global_ask_answer( + _Pool(), + question_text="synthetic question", + corporate_entity_ids=set(), + chat_client=_FailingClient(exc), + ) + ) + assert raised.value.status_code == 503 + assert raised.value.detail == ( + "Ask Agent is unavailable: contextual-orchestrator could not complete the answer" + ) + + +def test_global_ask_provider_failure_is_reader_safe_and_classified( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """Known provider/schema errors produce a provider-unavailable signal.""" + counter_calls: list[tuple[int, dict[str, str]]] = [] + + class _Counter: + def add(self, value: int, attributes: dict[str, str]) -> None: + counter_calls.append((value, attributes)) + + exporter = attach_inmemory_tracer(monkeypatch) + monkeypatch.setattr(observability, "_FAILURE_COUNTER", _Counter()) + caplog.set_level(logging.WARNING, logger="lineageweave.observability") + sensitive = "provider response body must not escape" + _call_ask(monkeypatch, ValueError(sensitive)) + + record = next( + item for item in caplog.records if item.msg == "lineageweave.server_failure" + ) + assert record.operation_code == "global_ask" + assert record.failure_outcome == "provider_unavailable" + assert record.error_type == "ValueError" + assert record.stack_trace == "" + assert record.trace_id + assert record.span_id + assert sensitive not in caplog.text + assert counter_calls == [ + ( + 1, + { + "lineageweave.operation_code": "global_ask", + "lineageweave.failure_outcome": "provider_unavailable", + }, + ) + ] + assert any( + span.name == "lineageweave.server.failure" + for span in exporter.get_finished_spans() + ) + + +def test_global_ask_internal_failure_is_reader_safe_and_keeps_stack_without_value( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """Unexpected defects are traceable without exposing their exception value.""" + exporter = attach_inmemory_tracer(monkeypatch) + caplog.set_level(logging.WARNING, logger="lineageweave.observability") + sensitive = "internal prompt-like value must not escape" + try: + raise AttributeError(sensitive) + except AttributeError as exc: + _call_ask(monkeypatch, exc) + + record = next( + item for item in caplog.records if item.msg == "lineageweave.server_failure" + ) + assert record.operation_code == "global_ask" + assert record.failure_outcome == "internal_error" + assert record.error_type == "AttributeError" + assert record.stack_trace + assert any( + span.name == "lineageweave.server.failure" + for span in exporter.get_finished_spans() + ) + assert sensitive not in record.stack_trace + assert sensitive not in caplog.text + + +def test_global_ask_source_gather_failure_is_classified( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """Source assembly failures share the Ask span instead of escaping as 500.""" + attach_inmemory_tracer(monkeypatch) + caplog.set_level(logging.WARNING, logger="lineageweave.observability") + sensitive = "source body must not escape" + + async def _sources(*args: object, **kwargs: object) -> list[object]: + raise ValueError(sensitive) + + monkeypatch.setattr(global_ask_queue, "gather_global_chat_sources", _sources) + with pytest.raises(main.HTTPException) as raised: + asyncio.run( + global_ask_queue.compute_global_ask_answer( + _Pool(), + question_text="synthetic question", + corporate_entity_ids=set(), + chat_client=_FailingClient(RuntimeError("unused")), + ) + ) + assert raised.value.status_code == 503 + assert raised.value.detail == ( + "Ask Agent is unavailable: authorized evidence could not be assembled" + ) + record = next( + item for item in caplog.records if item.msg == "lineageweave.server_failure" + ) + assert record.failure_outcome == "internal_error" + assert sensitive not in caplog.text + assert sensitive not in raised.value.detail + + +def test_global_ask_timeout_is_provider_unavailable( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """Timeouts are classified as provider unavailability, not internal defects.""" + caplog.set_level(logging.WARNING, logger="lineageweave.observability") + _call_ask(monkeypatch, TimeoutError("synthetic orchestrator timeout")) + record = next( + item for item in caplog.records if item.msg == "lineageweave.server_failure" + ) + assert record.failure_outcome == "provider_unavailable" + assert record.error_type == "TimeoutError" + + +def test_telemetry_without_endpoint_does_not_latch_configuration( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A disabled first call still permits a later operator configuration.""" + monkeypatch.setattr(observability, "_CONFIGURED", False) + monkeypatch.delenv("OTEL_EXPORTER_OTLP_ENDPOINT", raising=False) + observability.configure_telemetry() + assert observability._CONFIGURED is False diff --git a/tests/test_tepp_client.py b/tests/test_tepp_client.py index 667eedec9..b670e4485 100644 --- a/tests/test_tepp_client.py +++ b/tests/test_tepp_client.py @@ -55,8 +55,21 @@ def fake_transport(payload: dict) -> dict: def test_configured_transport_sends_optional_bearer_key(monkeypatch: pytest.MonkeyPatch) -> None: received = {} - def fake_post_json(url: str, payload: dict, *, headers: dict, timeout: float) -> dict: - received.update(url=url, payload=payload, headers=headers, timeout=timeout) + def fake_post_json( + url: str, + payload: dict, + *, + headers: dict, + timeout: float, + service_peer_name: str, + ) -> dict: + received.update( + url=url, + payload=payload, + headers=headers, + timeout=timeout, + service_peer_name=service_peer_name, + ) return {"status": "accepted"} monkeypatch.setattr("backend.app.analysis_run_start.post_json", fake_post_json) @@ -66,6 +79,7 @@ def fake_post_json(url: str, payload: dict, *, headers: dict, timeout: float) -> assert received["headers"] == {"authorization": "Bearer test-key"} assert received["payload"] == _sample_request().to_json() + assert received["service_peer_name"] == "tepp" def test_configured_transport_hides_raw_provider_exception_chain( diff --git a/uv.lock b/uv.lock index da3f4fd54..7d790b33f 100644 --- a/uv.lock +++ b/uv.lock @@ -167,6 +167,137 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/aa/29/35e016098c814cd93de9cd320c66b5bfba14dc6ecedd3cb518fa7c408c69/cffi-2.1.1-cp315-cp315t-win_arm64.whl", hash = "sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692", size = 186360, upload-time = "2026-08-03T21:21:13.636Z" }, ] +[[package]] +name = "charset-normalizer" +version = "3.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e5/3f/143b048436775b0f76ac3eec145c019e8173ccc2885c8f20319b996d5e83/charset_normalizer-3.5.1.tar.gz", hash = "sha256:6117b84ea48435e5356dc737f5121485c30920ba43375fa7b434fd753df0eac3", size = 171764, upload-time = "2026-08-15T08:20:44.807Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/30/27/78873dc8b6a56357517b74b6bb9568b80450e7bb4f6ef7e3fa9d22aa0bd7/charset_normalizer-3.5.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:5b6d1386bf0096d26d3a863dc0a487a5b4eb9aa93cf5ba69683d29dde6b9d60f", size = 344456, upload-time = "2026-08-15T08:17:10.072Z" }, + { url = "https://files.pythonhosted.org/packages/9a/4c/be49ada26b1f0232d57aa89bbebf997a5cc2332a5616b6eca26ff680044d/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4582c27e8c889d64811987b5967fbd3ae0c823fe1fd933b543d55ac20bb475fa", size = 238530, upload-time = "2026-08-15T08:17:11.563Z" }, + { url = "https://files.pythonhosted.org/packages/76/84/6f1290fa07ae6978d3960caa3eb1b8019bf9284ab7c2297b00c099ef4250/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1d1c7a53a6c2103925cdd6d7229f8c567379f211c869793df679f2e9f738c369", size = 230200, upload-time = "2026-08-15T08:17:12.919Z" }, + { url = "https://files.pythonhosted.org/packages/e7/a0/47b18adeed31c8f16ba9700f32c1b18594cfa09f47eb672a488c273c22bf/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e6621fb2a4988d6e53eedc455e5903e2679f3967b8acb3d639f1b63c14a2e893", size = 262222, upload-time = "2026-08-15T08:17:14.571Z" }, + { url = "https://files.pythonhosted.org/packages/38/fe/341861ac118dae06f3ec0eb487488af52128f2ef2faf0b11003944d22259/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7c0c10730342b0c9b35dd1d619beb8214e520bd96a1f870f452680b238aab3e0", size = 258951, upload-time = "2026-08-15T08:17:16.158Z" }, + { url = "https://files.pythonhosted.org/packages/6f/89/bb5108dc6c3651dca963f2b0a3ba19bbcb370c94e1b6d3e0e844a58e6dca/charset_normalizer-3.5.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b9af956078716df40d985fb0dfeb2c2120c5ca92ba4ff4b388acfd01cdc14d08", size = 248801, upload-time = "2026-08-15T08:17:17.683Z" }, + { url = "https://files.pythonhosted.org/packages/b1/ba/ef83ae3aca816393decfa3530976f38a79812d707b80b580ac33b83f9877/charset_normalizer-3.5.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f9f8405c2c758532c74fed975dbee57be1f31a6e865c031870c79a6ed3212ada", size = 244070, upload-time = "2026-08-15T08:17:19.191Z" }, + { url = "https://files.pythonhosted.org/packages/f6/0b/c5292a2462d69b7378ea89793bbb5b2b6fcf6f7dd6d1667f9619094ad553/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:96fef3e886d6a9874b14f27fc193fbdc69d5d8035783d86aa4e1cea594e695f9", size = 240110, upload-time = "2026-08-15T08:17:20.547Z" }, + { url = "https://files.pythonhosted.org/packages/46/22/111e5be3b740d5c2a5bfcedb3d237b6591e5c2e82ae9d6ffcb121fe0909c/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5d8531a6569d025f68e2321e7638fb7978f23db58e5f69f56913837aae03816e", size = 232836, upload-time = "2026-08-15T08:17:21.895Z" }, + { url = "https://files.pythonhosted.org/packages/f9/d2/d2aad6fe0dbb44b194bf3becb60f5a0ac48446ade999a47fe7bb41eb09a7/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:aae2ee51122d3ae968a3837d97dc24a0aeebb0dea23694422cd172bd30017cd6", size = 262712, upload-time = "2026-08-15T08:17:23.727Z" }, + { url = "https://files.pythonhosted.org/packages/35/5a/337e4663a5eae6de99db940ee8066d4145caafb61327db62deda15313cce/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:7235dc28fc6dd9d832ac7c7bce95367dedb85929f17368a0c2bee1e080b9acbf", size = 242977, upload-time = "2026-08-15T08:17:25.157Z" }, + { url = "https://files.pythonhosted.org/packages/ca/85/f82f8a92e31c7519410e2e1afdc630f28ec47490ce2c09a11c1a43cbb459/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:4abdc5f9ad448c1ecbfae2974b820535d6bc6e7eef63babbab3d81cf46968c71", size = 260207, upload-time = "2026-08-15T08:17:26.602Z" }, + { url = "https://files.pythonhosted.org/packages/b7/52/643d11ffd60e9ac2fd1fb87e167a19285b9eefeff4a40e63c87cbfbeab36/charset_normalizer-3.5.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ba501e667c17d8411f98e67a022d9604ef179aff0e459b7e292c796837c13573", size = 250562, upload-time = "2026-08-15T08:17:27.971Z" }, + { url = "https://files.pythonhosted.org/packages/62/16/46556278c2168d12df9da7fede5dc6fc70e60301b26a82bbeec238c9cfe3/charset_normalizer-3.5.1-cp312-cp312-win32.whl", hash = "sha256:cfa1c0cc3a8f9f53f1243a5a99ac36fd003880199383b37672e86ddda9cb07e2", size = 178507, upload-time = "2026-08-15T08:17:29.277Z" }, + { url = "https://files.pythonhosted.org/packages/9d/7a/4c6c298171e6b3e745633180ff59350fc0ca0db1ffd28df1e369e0579f71/charset_normalizer-3.5.1-cp312-cp312-win_amd64.whl", hash = "sha256:3617ac3cfd8b9888f145ad89dd6e692285834b0201c6074a5eeaad3fd4d668c2", size = 200551, upload-time = "2026-08-15T08:17:30.668Z" }, + { url = "https://files.pythonhosted.org/packages/cd/d7/eb95a042f0dd22e304b0b6472b154f3546a1a039a9ee89ccb2a7f61591fc/charset_normalizer-3.5.1-cp312-cp312-win_arm64.whl", hash = "sha256:88e85ab89cb822c1e635f51d6d32e488f94e002e70e2f492bdb8b945543f345a", size = 180700, upload-time = "2026-08-15T08:17:32.028Z" }, + { url = "https://files.pythonhosted.org/packages/bc/61/2cb6ad133dbbb449fa2d37ccae973232f4827e799af258d15e589a3d1e9e/charset_normalizer-3.5.1-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:4f298bdadb8f0b9e5672877f647d1be9373ef5320c9e2f049795e26cad28b6a9", size = 211584, upload-time = "2026-08-15T08:17:33.597Z" }, + { url = "https://files.pythonhosted.org/packages/18/57/a305c968be1ca13f3dd1b32f445877e97addf55d80b65c7cb35fac82b777/charset_normalizer-3.5.1-cp313-cp313-android_24_x86_64.whl", hash = "sha256:88ca277405c2d3b71c4e1c2ee0e7966e807bcba86a69d11e19ba199d18ae4491", size = 223359, upload-time = "2026-08-15T08:17:35.022Z" }, + { url = "https://files.pythonhosted.org/packages/09/0a/d3646670292ce8d8f8cc11ac067d44885e697a5591f57a9221128da5e7b3/charset_normalizer-3.5.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:9362dd90aa7dab48c0054a21187791ccf05473f7dba5d92b8033ae62164675e7", size = 194464, upload-time = "2026-08-15T08:17:36.452Z" }, + { url = "https://files.pythonhosted.org/packages/de/93/d51ec556e01042fed6f993ea859311bc7917b466684182fbbceb6ca24762/charset_normalizer-3.5.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:977cdbd483a9cff38179bea4fd754289a6f2195c7abd414aba85410b3e66cc5e", size = 197676, upload-time = "2026-08-15T08:17:37.819Z" }, + { url = "https://files.pythonhosted.org/packages/a4/a0/562247944386f7d4ef94467e84876600cc1e0f1b93239aaa9213d2bc3cbd/charset_normalizer-3.5.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:e90251c0c7bdd54a100a0dce3c07b7e637278c93af29dbf78ebb89a58c4bac7d", size = 340473, upload-time = "2026-08-15T08:17:39.303Z" }, + { url = "https://files.pythonhosted.org/packages/31/e7/1d994be1b93d41e9502b8b0460eaa88a1dd8df335df415db87d6c3e91ab2/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:94d78ecec2605a8d0398b0f365d5f12a63248438516f5dac536a5eff7337df4a", size = 240156, upload-time = "2026-08-15T08:17:40.66Z" }, + { url = "https://files.pythonhosted.org/packages/09/53/27923ce5cc6cbccb832037b27dca98882d9c53e9b69e866bbbef4aae7fc8/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:d59b75732e9b6f27388e10c14b0259cc5f2e48c78627d185e6a177b58ad3cffe", size = 228246, upload-time = "2026-08-15T08:17:42.003Z" }, + { url = "https://files.pythonhosted.org/packages/ce/48/5a97e84d63af1d55c07439cb80e56d99a8efb4295700eb4e18c0d1615d2c/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0d929fc574b4d6fd9e7c0f5c2ede8716a41911923aa7fa5fce38e0818aa4a1ac", size = 263660, upload-time = "2026-08-15T08:17:43.627Z" }, + { url = "https://files.pythonhosted.org/packages/7a/c2/071575791dcc88316c0a9a65ce38897a82e4cfe4a325f0f7fe1b1ac47bcf/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:394fea06235c8543390050ed5f529187074b029fb027213f6c46ac11ab5d950e", size = 260354, upload-time = "2026-08-15T08:17:45.094Z" }, + { url = "https://files.pythonhosted.org/packages/fb/af/63240b0c0248c075c2535a1f1bd992821d8251b9f173abc13329661d09e4/charset_normalizer-3.5.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:62b55f6722735a6c472f88361cde6640608773d9443cebdbb51abf436a1fcdd3", size = 250638, upload-time = "2026-08-15T08:17:46.496Z" }, + { url = "https://files.pythonhosted.org/packages/4d/66/70dfad64f15be09c15ccfee81330a7e515895dbe296dd23114e9a231268a/charset_normalizer-3.5.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fa48b1b63d639f9483e0633e092f5851e2348c352f1f9bb6c8182f87884ef876", size = 244583, upload-time = "2026-08-15T08:17:47.963Z" }, + { url = "https://files.pythonhosted.org/packages/c0/24/ef36367d38b9ddd4bccbf72888c342e8de1f5ae506fa0b2dcf970e2732a1/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c71fb0d56c920c269cd3e2e3fe7c610e3f1fdb21a6ce60efa6430ff63676cea6", size = 242038, upload-time = "2026-08-15T08:17:49.481Z" }, + { url = "https://files.pythonhosted.org/packages/db/ab/55e683ba0fff2e43adafc10daa3001eac90fdaa419a97227d5a7067eedde/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:485a0d363cafefcd2538a73c7c838daa2035f09b2c9f9b5e3133f80c6aeb84c2", size = 233677, upload-time = "2026-08-15T08:17:50.845Z" }, + { url = "https://files.pythonhosted.org/packages/bd/67/0f40eaf8d1b6e7cf15e82382a2965efaca787fc1c2794b7021d37aaf5036/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5c0ea61a470e070686aa30892fed79e297d2c8d0ab46b8bcdf027d38c51da591", size = 264491, upload-time = "2026-08-15T08:17:52.61Z" }, + { url = "https://files.pythonhosted.org/packages/5c/64/12b4c2a11ee8df4fcc518c78b0d93e3a92bd3d5253d1617ce74ff0e8c7ef/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:90b7481fb62fbe172c558bc6fd1c4c98d82004a54a7551f20e11ac9bf0b8708c", size = 245196, upload-time = "2026-08-15T08:17:54.023Z" }, + { url = "https://files.pythonhosted.org/packages/37/2e/651d910af6d0fba325eee1cda37ec5443462ed25360e666c144166eb6091/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:35fe081843b35aad20ffeccec3eeffbe637b15d14f3fb22cc1b59cd8ec17e93c", size = 261660, upload-time = "2026-08-15T08:17:55.491Z" }, + { url = "https://files.pythonhosted.org/packages/90/c6/b09e05e6db7f64338e0dc067c79577b1138da86c1e38369096851d96be88/charset_normalizer-3.5.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fd0350afdc3aabd5576f60ea109228bd5538139713c7b094c5cd27c73a98bc6f", size = 252618, upload-time = "2026-08-15T08:17:57.025Z" }, + { url = "https://files.pythonhosted.org/packages/76/4e/362d4f9fdcdf5556fb2aa3ce7d4a58ebce03ed1ff03aa1d9aca8d02f13f3/charset_normalizer-3.5.1-cp313-cp313-pyemscripten_2025_0_wasm32.whl", hash = "sha256:9d9a0dc7cbe9bec24c3f767c9122c41fe5a1bc43f47cd099d00d393e09769de4", size = 140362, upload-time = "2026-08-15T08:17:58.425Z" }, + { url = "https://files.pythonhosted.org/packages/b4/d4/703be739b26acce318bd29eb3b25b7209e1b1f527f9eae3d1f1f01fdde2b/charset_normalizer-3.5.1-cp313-cp313-win32.whl", hash = "sha256:d63600d620ad0064c3a748b950ac5ea38a80190e5498532efefa4b7b3f1da1f3", size = 177755, upload-time = "2026-08-15T08:18:00.037Z" }, + { url = "https://files.pythonhosted.org/packages/8a/33/56d97ade41c8db611e727168c52ae46c9224c362ec28d4b65d7e9869e8da/charset_normalizer-3.5.1-cp313-cp313-win_amd64.whl", hash = "sha256:aea996a6aba25260827c9ea511d1addfde2da9eb686ac961838509086188b7e6", size = 199295, upload-time = "2026-08-15T08:18:01.506Z" }, + { url = "https://files.pythonhosted.org/packages/5b/75/5b20dd1e6573a01a08158fe104104fa2c8abf941745596954185726cd46c/charset_normalizer-3.5.1-cp313-cp313-win_arm64.whl", hash = "sha256:fd0a274c0e5f9a21565cd9d3dd749b61f96b7aa1e20a93aa1ba4029518f2e5c0", size = 179856, upload-time = "2026-08-15T08:18:02.929Z" }, + { url = "https://files.pythonhosted.org/packages/29/cd/2b812ce5e888f1ce69a5350281e58aab07ae64a958ecae8912f30865718e/charset_normalizer-3.5.1-cp314-cp314-android_24_arm64_v8a.whl", hash = "sha256:774d157f112367ff4abd29019f38f023c24e00e56edc7829c20e358a5a913ad8", size = 212318, upload-time = "2026-08-15T08:18:04.403Z" }, + { url = "https://files.pythonhosted.org/packages/9e/4a/a6ee107430768a5334e6d63f31f148a04a1a491ef161a1ac9415a73f2fa8/charset_normalizer-3.5.1-cp314-cp314-android_24_x86_64.whl", hash = "sha256:26422d45fd13551cf564c58932f7d72b4f58b93b0fcf18c35ba6be12b46bb102", size = 224897, upload-time = "2026-08-15T08:18:05.997Z" }, + { url = "https://files.pythonhosted.org/packages/c3/d9/35ae3f64f29d0179c35c3baefe575904df2913dde519129c7f75995a2b1d/charset_normalizer-3.5.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:09a7bba9f739468c8e78c36a75c33768e53cb1959fc638f510454c14683f00d5", size = 194848, upload-time = "2026-08-15T08:18:07.397Z" }, + { url = "https://files.pythonhosted.org/packages/74/76/f2fc7380f056cc273a53af37f50d08ad54b2c59f61078f31432edcf1c2bd/charset_normalizer-3.5.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4c9548dc78002099910abaebc0a72ac58b7d30931869e0351c09b507dff4ece3", size = 198163, upload-time = "2026-08-15T08:18:08.989Z" }, + { url = "https://files.pythonhosted.org/packages/e9/40/095ce62fa078483cccc1fa2b36e6bc9580b85422a20ee9f925341c50e44f/charset_normalizer-3.5.1-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:c428c6c31eb5f4277d7f8eccaf767fbd548ddd5ce3c8b4f4cbbfab3d96b5904c", size = 341823, upload-time = "2026-08-15T08:18:10.458Z" }, + { url = "https://files.pythonhosted.org/packages/f1/5a/0e58b1c04a1596e0256f407274a92d5fb2ee21324409d1fab1da48a65b5b/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2f06b7eae9dbe77fe1d644ca244dad508de8d302870a43f3c559b521270938a0", size = 242458, upload-time = "2026-08-15T08:18:11.989Z" }, + { url = "https://files.pythonhosted.org/packages/22/95/b4618ce912e6db0b1aae89ba788e38e8a7eba0f3025cc66e8c0699f977b2/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6b7430cf5728e68f6c462254009a6ef4086e1bea43cf2f57aa9c55fb4f50ff96", size = 226717, upload-time = "2026-08-15T08:18:13.401Z" }, + { url = "https://files.pythonhosted.org/packages/8a/76/c681192bbda3d55356db5dadd64381d5202b37c6b598fcda5282e88b5d3d/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ab743e9bc90c1f73552ec33e10e3331315acd2c397b36065b591b0181de533cc", size = 266111, upload-time = "2026-08-15T08:18:14.961Z" }, + { url = "https://files.pythonhosted.org/packages/88/be/55127bfca72c0cff6c022488d140d7c5b04c771e3b72e9bdb4836d54979d/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f6f7deae3feb4edfa2efaf7c574fe88cbf055038a6abdb40188e4fff66d5699f", size = 263128, upload-time = "2026-08-15T08:18:16.515Z" }, + { url = "https://files.pythonhosted.org/packages/e0/91/39c3af510b0aa32bbda03374259200f28430febfd1bf5e511fe765282ce5/charset_normalizer-3.5.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:15f024313246a4ed976c60f440bb8d257815513a681d212ff74fd46f7d715a90", size = 251240, upload-time = "2026-08-15T08:18:18.127Z" }, + { url = "https://files.pythonhosted.org/packages/1c/a5/cbe418bbc6ecdfc3e05a0116002897c4b403a5e838d697e64c78e9f0190d/charset_normalizer-3.5.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:823f82903d189af463d7df250ef1f7f696f3cee08cc8d91deb565e8d425f6506", size = 245282, upload-time = "2026-08-15T08:18:19.625Z" }, + { url = "https://files.pythonhosted.org/packages/cc/a4/689bb42e8e7cd492f3cb64907c6bc00ad247ec9a3628cd3f8eed126e8ae1/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:01e93745f7f219b703b60ba7afead36cfc4242782be5af484673fc500df12da5", size = 244597, upload-time = "2026-08-15T08:18:21.121Z" }, + { url = "https://files.pythonhosted.org/packages/c1/ce/9962938e179cf9f699d3f1e7b3114b5d7642dee6a893745229f9dd04f274/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:329fc3ccb63ad22d867d84c2adea759a64079a37ba4a343433b02c7a2816871e", size = 231376, upload-time = "2026-08-15T08:18:22.57Z" }, + { url = "https://files.pythonhosted.org/packages/85/54/46000450ada53bd9eac5429a2c8c54cd2d9b39c0c255f229aea9af0948a5/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:bb57753e36e4855b8ca375069482250a6246372331a3e4f3407eaebb007443f5", size = 266715, upload-time = "2026-08-15T08:18:24.235Z" }, + { url = "https://files.pythonhosted.org/packages/3d/bb/618749d70f792b44252a777bf89bfb86823b9bbc1ea13fe8ce759b07f38a/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:fce8cbd4997efeb450bd298b54f755dcdff18d496f7a5ddbb4867c6d7c88fdc3", size = 245848, upload-time = "2026-08-15T08:18:25.726Z" }, + { url = "https://files.pythonhosted.org/packages/7e/3f/ffb64458527c7668031d5eb095d978de561958dc9f5b53f8e488a533e603/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:6c9cdde8becb25a7fde49924511aa2644d6f8081cc8df8e9452724303348d8e3", size = 264521, upload-time = "2026-08-15T08:18:27.193Z" }, + { url = "https://files.pythonhosted.org/packages/4f/ab/74a55fd803916a35ac461daf002708191aac19b546b80dc8cabfedc63d98/charset_normalizer-3.5.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:9ac4444d8d4fd4c4bd08bf451ed3167aa9e7ec6cdb41b648794f1d1103652e36", size = 253054, upload-time = "2026-08-15T08:18:28.568Z" }, + { url = "https://files.pythonhosted.org/packages/a0/2a/6a9034b7d3c60b17499afb482df5878bf9fa20b50cc3887d5ef017a833db/charset_normalizer-3.5.1-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:f03ac127268b43ef4fe9e6ab6794a6794b49485a0cc0c1db79876d2f33f75bc7", size = 140580, upload-time = "2026-08-15T08:18:30.214Z" }, + { url = "https://files.pythonhosted.org/packages/f3/46/1d362e1a00d035d66b9869e1281eee115907f7e390a16a07824ab5737360/charset_normalizer-3.5.1-cp314-cp314-win32.whl", hash = "sha256:1f5883d77fd409a261abb5dc8ccbe335720d798b1de4abb3b1d47ccbbc76b53b", size = 180325, upload-time = "2026-08-15T08:18:31.877Z" }, + { url = "https://files.pythonhosted.org/packages/7a/7c/4938c329b6a9d446f6a59aa2092ff7118f274209b5ed0e26893d1d30a63c/charset_normalizer-3.5.1-cp314-cp314-win_amd64.whl", hash = "sha256:c658c50ac0c98cd755a2dd50b7977d3bca7df401dcc47fbdfa87db53ef7d4e8b", size = 204175, upload-time = "2026-08-15T08:18:33.466Z" }, + { url = "https://files.pythonhosted.org/packages/ac/33/eeb384dbd8dec570661354592f4f2e1b2fcc92585624d146a000caf53841/charset_normalizer-3.5.1-cp314-cp314-win_arm64.whl", hash = "sha256:4bea7f8ebe90bbd7f0e4a2de42ca6924ba23e3e76418c408ff82f1d46fabd687", size = 184123, upload-time = "2026-08-15T08:18:34.913Z" }, + { url = "https://files.pythonhosted.org/packages/1c/6c/c73fa9d5a85f6ab05395de61c5f6984e0a9ff40bb5ff888d46dff02526c6/charset_normalizer-3.5.1-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:fbc597639158fd7c14d55e808718848319540f51b0e6746e3eefa59723a4a348", size = 381682, upload-time = "2026-08-15T08:18:36.349Z" }, + { url = "https://files.pythonhosted.org/packages/30/c7/63565f860921457feba93bae6c86fb7746deb4cffeed2f375cb845318146/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e71c909f353863b2b89c83de2ebed71ea6d0df8a6ef65a128193c5e650766bef", size = 240826, upload-time = "2026-08-15T08:18:37.887Z" }, + { url = "https://files.pythonhosted.org/packages/06/ae/7ae8807410dfa33f8e6f1715740adeaafa8a816cc4cb33508f54b1f7c896/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:7ac76cf9afd34929d76eb7fcb63be476a4853d8a96f0dcf2d0db68a0cbdf9885", size = 227861, upload-time = "2026-08-15T08:18:39.315Z" }, + { url = "https://files.pythonhosted.org/packages/e9/a3/887c1642f0da26000b0e0652d91071113c0e72cea33952e225cf589f49a9/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a3a370082ce34d0612f421e15fe011c53bb1feff21a26d06ad4fb244dab5a375", size = 260758, upload-time = "2026-08-15T08:18:40.88Z" }, + { url = "https://files.pythonhosted.org/packages/3e/11/e6f5b9a3d0e55b0ef7505cd3765cdd48f22db89994c947b316f52f801fd8/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:256dd4d85d9e4dc595e2bc983c980e73f62ddeb3165c58b4c3dfe78c5c8548c1", size = 259950, upload-time = "2026-08-15T08:18:42.351Z" }, + { url = "https://files.pythonhosted.org/packages/1b/ee/e4e10a94d51cd1ee638aa7e00b65399e6b2a4e8376ab6d2eac9f95586671/charset_normalizer-3.5.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:58d4aa13a59c969dbfdf9e6a9560e242cbfd9e8a8f50c2747714df1a423adf65", size = 249329, upload-time = "2026-08-15T08:18:43.914Z" }, + { url = "https://files.pythonhosted.org/packages/c4/25/d5f4198819e6059735a84e8d0bfb72dc33976da67b97adcd3fb5a5e07ec6/charset_normalizer-3.5.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0c6dfb5ca6723eeed15aa8e564a014d69fcb8812f94eef11fe3631e0508199f5", size = 243137, upload-time = "2026-08-15T08:18:45.368Z" }, + { url = "https://files.pythonhosted.org/packages/a5/e9/e925ca7569cf9fb9701fd82503fee73eea5268fdb856bdd64947092d3daa/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:c010f5581d9c612804cc59fcf7b524b707fbcb72828551237ab545bb5c7034af", size = 242820, upload-time = "2026-08-15T08:18:46.842Z" }, + { url = "https://files.pythonhosted.org/packages/34/17/672c251a888ed2aebcdd2fe830ad0104e25ff83c43f5c4f9c15e9fc6853c/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:52ec005752a56ae79547a05c0139ca2501a0c866390b6115008456b9f0e7cde1", size = 230504, upload-time = "2026-08-15T08:18:48.353Z" }, + { url = "https://files.pythonhosted.org/packages/3f/fc/f6a85abebd42ce4da2f1db0aa56cc6a0df1995e318b3875d14401b8381d1/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:2bced4061f000f7187254a02ad3433ae17eaf991747ceea2f478422590a5bba9", size = 263087, upload-time = "2026-08-15T08:18:49.859Z" }, + { url = "https://files.pythonhosted.org/packages/98/66/7c42677e739ba66746b297e2046918d793078094dc239e1e72768cffccc6/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:9eea3ab2597a5e65fe65296e2d6a84570845a6b55532d90333d740d48bbc850a", size = 243269, upload-time = "2026-08-15T08:18:51.601Z" }, + { url = "https://files.pythonhosted.org/packages/de/d8/a50b79237f417af10f8c2a501ce8d1ca87829a22e69117891ca4ba20a69e/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:496846868fea80e479324862fa877f02411f2fd0f83b79ccee2607aa68b2a032", size = 258766, upload-time = "2026-08-15T08:18:53.23Z" }, + { url = "https://files.pythonhosted.org/packages/2e/1d/0fc91aeaeb3c83b748f532399ce67cf84604b48297405d740000f7a9e786/charset_normalizer-3.5.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:85d5855daafc240cc045c026d7a15fd198a09b0fc8ff6f5ecbb5297b509cb11e", size = 250814, upload-time = "2026-08-15T08:18:54.768Z" }, + { url = "https://files.pythonhosted.org/packages/ae/10/3d8c777cf9024615295aa1b808324ad5b4a77855869c00824bad74ffaf8a/charset_normalizer-3.5.1-cp314-cp314t-win32.whl", hash = "sha256:58d3e12c88e0950bca850ae1f7c256055c097639c2edb9eb123af9807d8b15e4", size = 191074, upload-time = "2026-08-15T08:18:56.305Z" }, + { url = "https://files.pythonhosted.org/packages/4d/81/ae557d3c44d1a1d688696d60563413a0866a91b7ebc50f20df838be3d8c8/charset_normalizer-3.5.1-cp314-cp314t-win_amd64.whl", hash = "sha256:acaf604462bf330b0d07e7a07c1d6e4adac79e5fb13e9c5140590542cafacc00", size = 216476, upload-time = "2026-08-15T08:18:57.889Z" }, + { url = "https://files.pythonhosted.org/packages/27/e9/61c01fb8b804692569c036b3fc50495814502dcf13a60649c6055390b02c/charset_normalizer-3.5.1-cp314-cp314t-win_arm64.whl", hash = "sha256:fdb8a068947befafba9952162645dc2fecaeb400e64584829ed5e9b2fbe21a7f", size = 194115, upload-time = "2026-08-15T08:18:59.418Z" }, + { url = "https://files.pythonhosted.org/packages/4a/4e/8544831ef59d8f27ce92c80871380fdacc8076a8a56ed62f82e54f991333/charset_normalizer-3.5.1-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:9085f87b0e38a2b92b8923059b4e8789fe40d9279712d15dcc670048d77079af", size = 342048, upload-time = "2026-08-15T08:19:01.054Z" }, + { url = "https://files.pythonhosted.org/packages/7f/a6/e3b46852424246065355644f4fb6dbccc0239a42a2eee27ecfc8957f0bcd/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2679de311c7946dde5d3b6f44941844133ff5c7cb86099c0061ab1e8901c20a8", size = 242997, upload-time = "2026-08-15T08:19:02.492Z" }, + { url = "https://files.pythonhosted.org/packages/03/3b/0cc9a26777334ab2f2e3089b948bbf4e4fe72ea70b897715ef6415043ec8/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:baf3775a2635e5a11fbd5e4e64ee69c7e86875d224a5c72aca4c141064589a90", size = 237014, upload-time = "2026-08-15T08:19:03.943Z" }, + { url = "https://files.pythonhosted.org/packages/8c/c2/027335f0aa337a2a2e121bac1ad88c4f02ba6053ea0926802784f3db11af/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8ac8c94b6539074e0f40899301273ac8402b9b3e01c7b7ba269ff30340aaaf20", size = 266174, upload-time = "2026-08-15T08:19:05.598Z" }, + { url = "https://files.pythonhosted.org/packages/86/d3/e367787febe4e74769dec0f406f2c3c8d1b955fce5aee1fd0f94e8367a45/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8fe532b3c966d1fb794e0698e4589d0444017ae77fc0b31edea13c0e35bcc449", size = 263361, upload-time = "2026-08-15T08:19:07.251Z" }, + { url = "https://files.pythonhosted.org/packages/af/3d/391b193eb9f3e84b02f9314088c386debdc0debee843535aaea2e2c6715d/charset_normalizer-3.5.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5c84bec0ab5ae0c64bfe73a7d2adcb5ce73b467523fc27fd6a28ab2aa6cbe35a", size = 252143, upload-time = "2026-08-15T08:19:08.816Z" }, + { url = "https://files.pythonhosted.org/packages/2e/57/de221f1745a90d418199761967e2776bfe2c275a1194220985e8c1d37833/charset_normalizer-3.5.1-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:854066be00447fa8de2ccbbe893e2ffc4b123ef16d897af794c1e18bd4a714b0", size = 252086, upload-time = "2026-08-15T08:19:10.255Z" }, + { url = "https://files.pythonhosted.org/packages/c8/e3/d119f86a01f9331e8186175f24873b1d74a7ee9e2e4b4d68f9947dae5afd/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:21b82d8082f6f5e7f456ef0bd16323d08de1266efbfeb476e64b2a91d1471a4e", size = 245231, upload-time = "2026-08-15T08:19:11.807Z" }, + { url = "https://files.pythonhosted.org/packages/26/de/d8e48c135ae480879539cdb179c8d3b50c7879497d75dd899b5763b69cee/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:838648accb3a7fd9803fd45c87bce8509648eb0c11bc34e216141300977244f2", size = 241546, upload-time = "2026-08-15T08:19:13.416Z" }, + { url = "https://files.pythonhosted.org/packages/67/c4/217755fd1abc50d326c252922cd642002758095a81ff45010337b8b3ef65/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:195ce897c6153c0700078142cf8efe3e6454ca4cf4357499e4078dfd83396626", size = 267033, upload-time = "2026-08-15T08:19:14.981Z" }, + { url = "https://files.pythonhosted.org/packages/b8/d7/34d8e404e358d2adcc5a228c2134643af00104c8fb0bf525f3688d756f05/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:978eab16f55b4ab2c2a745be9a0a840bf8f09a7f227d9c76eb30214d078865a5", size = 252045, upload-time = "2026-08-15T08:19:16.618Z" }, + { url = "https://files.pythonhosted.org/packages/5e/fa/40414471acf0aa0692ca77305aa00e434fcd8288f0941c93c30e9a5f8f2f/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_s390x.whl", hash = "sha256:cc0329df4caaceb950d2f580b5ac716a377f7059624a0bafaeaf8a218c6ed774", size = 264866, upload-time = "2026-08-15T08:19:18.101Z" }, + { url = "https://files.pythonhosted.org/packages/32/90/fcc850bae791abd2e0c041847f13e270aa08692a79f3e00de6d2dce1cb50/charset_normalizer-3.5.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:687c9ca3035544b113bea2055e180af96fb63c0c476e22a9180f51925186e7b7", size = 253932, upload-time = "2026-08-15T08:19:19.734Z" }, + { url = "https://files.pythonhosted.org/packages/af/af/53afe99068b3c10b4cbae592a52ef72a7c92c0188440e83ee3a078fd8f75/charset_normalizer-3.5.1-cp315-cp315-win32.whl", hash = "sha256:706bfd38730a5ac7a365793269a00f4e988178cec121391f4248d84ad8c972e9", size = 180320, upload-time = "2026-08-15T08:19:21.37Z" }, + { url = "https://files.pythonhosted.org/packages/c9/bc/f46a132041b29e4a8779ed712d3df1bf112e94ca8de58b66d7ec2c0cf8b9/charset_normalizer-3.5.1-cp315-cp315-win_amd64.whl", hash = "sha256:92caef967d287a407085d61176fce4012b1dd62daed4eb6d5ceb26d3d2538712", size = 204174, upload-time = "2026-08-15T08:19:23.088Z" }, + { url = "https://files.pythonhosted.org/packages/a1/5d/9ed554480eda8e447b673648628fdc29574d23dbad01fe11837adedd1cae/charset_normalizer-3.5.1-cp315-cp315-win_arm64.whl", hash = "sha256:5fc45d653ea8c9a20479167e11d4a0f8cb2fa3470737ab6f9c827532313187b7", size = 184126, upload-time = "2026-08-15T08:19:24.471Z" }, + { url = "https://files.pythonhosted.org/packages/3b/32/9b8929bf384061ee1fe5d9c27c6f9776d3d824039ad4e14c88ec00c7808e/charset_normalizer-3.5.1-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:59171c6e45bf07d0d5cab3b0bf81d945035530f6873398b3b531c31184d46663", size = 381441, upload-time = "2026-08-15T08:19:26.038Z" }, + { url = "https://files.pythonhosted.org/packages/96/10/e9aa7923d3ddac652c99a1c5f7be494e737e151566a44abe018daf757f2c/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9dbdd9205662134957cf0c324f639bdc5031c0ca056e2369e238db75187c0f11", size = 241742, upload-time = "2026-08-15T08:19:27.532Z" }, + { url = "https://files.pythonhosted.org/packages/28/53/a2d249ebddf47b889a100c0bdcb61a2f9dbb8bc24ef325cc062e4f476877/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:e4b018dc5a0eee4676e38fe84a47a427816c590b93b55d9025274ec4d6ffc2dc", size = 235298, upload-time = "2026-08-15T08:19:29.274Z" }, + { url = "https://files.pythonhosted.org/packages/7d/07/469f78af590f7d5cd48e20d8dbfa3d66deeff9ba37768c04d886b5afd45c/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ced3fdd71aaa83ce593746c2edb42b7a59cb4c19c8b5c407781c72e493aae55a", size = 262500, upload-time = "2026-08-15T08:19:30.955Z" }, + { url = "https://files.pythonhosted.org/packages/55/66/3bb56a47f7dcba014055b1a1d33c6f08bbe9c1e74dba154cfa25f90ae885/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:19a3dd5aa73cef1c99687c4fc57db016a9c17104ae1185da88ba566a5d3bebe4", size = 258888, upload-time = "2026-08-15T08:19:32.458Z" }, + { url = "https://files.pythonhosted.org/packages/ff/c1/2adc2800903fb013210349313b710a5376856578d9e33e6b9a1d8b36714a/charset_normalizer-3.5.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cc5d36d96478aa9c60654bd932525bf32964c62a7281eafdf16d85003a8d6004", size = 250243, upload-time = "2026-08-15T08:19:33.94Z" }, + { url = "https://files.pythonhosted.org/packages/95/b5/a18d0dd1157ab655cc2cb14a545f4a4784bbad70ab3502412e36097502d9/charset_normalizer-3.5.1-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:04368edf83514385ffc3e1cfd4546e595f4f1272dd23ba437a93a9cc3741d47b", size = 249871, upload-time = "2026-08-15T08:19:35.413Z" }, + { url = "https://files.pythonhosted.org/packages/ad/c3/525f508cd1e58d0450ac55ed40ac75bc3a97482c59def5278456a5fbf03c/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:9b5db6052055d34d41230fb78d7c439c23dc536a9896f6cb039e8dd92cfc1263", size = 243580, upload-time = "2026-08-15T08:19:36.886Z" }, + { url = "https://files.pythonhosted.org/packages/7c/c1/49a91fe7e97c8140094ca5c64161ab623a70d9f636bf834eace14048acb5/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:252d099029bcbea642f2a06c4ed5046bdf8b5a8150b64afa5e027e88b106e5ee", size = 239807, upload-time = "2026-08-15T08:19:38.392Z" }, + { url = "https://files.pythonhosted.org/packages/d3/58/56a48c296601274c4689b864a8e2dfb209b81dfcb39472753ce95eea662b/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:6199d5606e2bbf2b096cf64d03f8b6790c91081d5ac866b8e7bb6422738cc60c", size = 264083, upload-time = "2026-08-15T08:19:39.856Z" }, + { url = "https://files.pythonhosted.org/packages/10/4c/dc48409274a1817ff349711d26c62aa0c597df865d4d69ef79160c859193/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:77efcff2b23071c349402ac1066667a3d011f62398d81408c9b88ad991747c9e", size = 250317, upload-time = "2026-08-15T08:19:41.53Z" }, + { url = "https://files.pythonhosted.org/packages/81/58/d325912115caec62d6bdd77bbab5e0b7da5d234a9f20affdffcbcb530d0b/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:a5cbd90ecf0fc62e64726917ad083b73001f0563657a87ec3c0b504e277dc90d", size = 258173, upload-time = "2026-08-15T08:19:43.07Z" }, + { url = "https://files.pythonhosted.org/packages/34/f7/b13b1ccae2c8ec63980d13be1890eb73f8aeabbfce02a24aabc0908788f5/charset_normalizer-3.5.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:4d26f14f041e83dd8edfd61f4cd4fa7285d31798b5bf1f28e70c367ba6c41d61", size = 251960, upload-time = "2026-08-15T08:19:44.587Z" }, + { url = "https://files.pythonhosted.org/packages/1e/25/ed3f9919c5aef8cc818be1f972f565f7610d7b2076b8ebb98839516ffc3c/charset_normalizer-3.5.1-cp315-cp315t-win32.whl", hash = "sha256:ac13b004224fb341e1e25a1ed5e19d32f57cdb2a403e01f003b46f051a550f6f", size = 191186, upload-time = "2026-08-15T08:19:46.293Z" }, + { url = "https://files.pythonhosted.org/packages/69/d5/43c2b3e9d8267092b913eb8b0603f0f71993c395632886bd37a7223f96cf/charset_normalizer-3.5.1-cp315-cp315t-win_amd64.whl", hash = "sha256:35aea775dc2bd5f54cd84a1cd2696cc3207c479cb9cf0bd346f0d343e4300ddb", size = 215947, upload-time = "2026-08-15T08:19:47.853Z" }, + { url = "https://files.pythonhosted.org/packages/a8/76/9aad3e9c8865e5e0efa9a7f6f81c37a67635a985145ecd44528a81e088ee/charset_normalizer-3.5.1-cp315-cp315t-win_arm64.whl", hash = "sha256:fb78f6e7fcd8ad785d28cd577168bc1aaee827b25bb8755638f694794ea98f0a", size = 193909, upload-time = "2026-08-15T08:19:49.383Z" }, + { url = "https://files.pythonhosted.org/packages/5b/97/fb4e82231aba271ffd775a1b4993b0defc4e3059f286ae41d9433409fe85/charset_normalizer-3.5.1-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:41876ee62a3dddf48ff1121ad8f0798032aa03f2fd35f21f34a4cab14f18d8d2", size = 331467, upload-time = "2026-08-15T08:19:50.959Z" }, + { url = "https://files.pythonhosted.org/packages/9f/2f/fe3f187327aac18e2d54e9d2b08e15d27bf9b642d9e51c219f130fc34d1a/charset_normalizer-3.5.1-cp37-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a6dac12ff6b846103483683f60c5f8fee205121adc58ffd87e90a90a3af69e99", size = 253057, upload-time = "2026-08-15T08:19:52.654Z" }, + { url = "https://files.pythonhosted.org/packages/d7/c7/9e48cee5c161fe24da823b61bf381921d77cb994a0a4de148e95018c1984/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:cee5dd7c6fb5dd52a0fe2a740f9bc6e3593f5f8b1788bde49de02086f30182b2", size = 240930, upload-time = "2026-08-15T08:19:54.163Z" }, + { url = "https://files.pythonhosted.org/packages/49/e0/716601f3cc69be7b198951150c75ead1ece33c3c8036ff6ffa46029659a0/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:343fb4f2821043bd87095f7b08a1a181febc8e36ac64212143bbfd0a0e1bc235", size = 230822, upload-time = "2026-08-15T08:19:55.807Z" }, + { url = "https://files.pythonhosted.org/packages/d3/05/71bfc5caa0abcc45aea1f6a4d50ac68e59605ddc7666fe8494f4cd229665/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ae4a097991662cd4fff0ddc74e0fe7874f82e00042fa0ea00855645ed0c79598", size = 260037, upload-time = "2026-08-15T08:19:57.312Z" }, + { url = "https://files.pythonhosted.org/packages/c3/92/de7e32ed05341e7a9c4c877c318418197b7f2d66a3b68d561bf2ac57ca3e/charset_normalizer-3.5.1-cp37-abi3-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:4b599739b93b2cbeded49645ae3c8d1405c29ddfbceac1545c87a3f9580a9e96", size = 255097, upload-time = "2026-08-15T08:19:59.056Z" }, + { url = "https://files.pythonhosted.org/packages/f5/7b/ade0a122600319dfa0b1000ab0f9731c94a817904cf3c5de408c73a4ede7/charset_normalizer-3.5.1-cp37-abi3-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b39b69b347e5e47a3b5b8cfc005c68c1ba347474e3960236c4944a8ecd174962", size = 250166, upload-time = "2026-08-15T08:20:00.612Z" }, + { url = "https://files.pythonhosted.org/packages/75/9c/019fbb9f4834491a160951349b1a3714439376f66e5f7cf18b4f18f0c7aa/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:a2028475ba855475b8b4d3cfeb4994269c967aea8b9892dfba907f4263a863a3", size = 241821, upload-time = "2026-08-15T08:20:02.321Z" }, + { url = "https://files.pythonhosted.org/packages/2b/b8/11d4840bfc99330cc7fbcc2681ee5a044553a6e77655508d8f9b2bff7b34/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:36047af20e17097c3bb9476c2b7655f2f7aa51322c0ba58c07695bedf755a950", size = 232529, upload-time = "2026-08-15T08:20:04.008Z" }, + { url = "https://files.pythonhosted.org/packages/18/96/2b3a21492d9f65171ac75d872f5018260013d00bfa0ff70ec9f179148cbd/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:4c4fb141a727957c93edfe5c32a26ceb6b5f6461d67146e2d39f51e16170bea8", size = 260348, upload-time = "2026-08-15T08:20:05.877Z" }, + { url = "https://files.pythonhosted.org/packages/d6/aa/a69a2028e8bd052476c245460ab19d7de595de084dd968f2d75cd50c3e25/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:2f293479cce755c75f1697e87c409b7ae4c555c7dfecb6e988ad13abba943031", size = 247234, upload-time = "2026-08-15T08:20:07.487Z" }, + { url = "https://files.pythonhosted.org/packages/35/8a/3d130aeabcaf3d2466af76b7b141c08d9e89c9016ab4b7cdd0f7dc2d1c62/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_s390x.whl", hash = "sha256:3588e376b3ea2eea84976f67273d679f229e24c66dce7b82ae45aef04ff6e072", size = 256917, upload-time = "2026-08-15T08:20:09.142Z" }, + { url = "https://files.pythonhosted.org/packages/80/c2/a7379b840292d0c1ab9fbd17d1f3967aa81794dc95bc74be8999d7fedcf7/charset_normalizer-3.5.1-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:e199fb99720074809a7720f1c0b4d919eea8b87e88713e0f8f602f7bef543d9d", size = 254846, upload-time = "2026-08-15T08:20:10.727Z" }, + { url = "https://files.pythonhosted.org/packages/01/65/d43b714731bb2f40d4053dfa00ecfc1c5a301f8e3316c5db3a09af59fe94/charset_normalizer-3.5.1-cp37-abi3-win32.whl", hash = "sha256:dd732602a7009217f658d5863d12d79d373a4de0eebc111094bcdd3bb8e0a6cc", size = 174216, upload-time = "2026-08-15T08:20:12.334Z" }, + { url = "https://files.pythonhosted.org/packages/35/4f/b911ed898b26a09789eba9c9200c999aff6c61b4bafaf4838e56d1a1e1a3/charset_normalizer-3.5.1-cp37-abi3-win_amd64.whl", hash = "sha256:70055ff39b97c99e7ae40ea3e393fb62aa2e44dbd9b29f8d14f42fb0025c3959", size = 199764, upload-time = "2026-08-15T08:20:13.908Z" }, + { url = "https://files.pythonhosted.org/packages/f0/a7/920baf467bfd9bf689f3b318340f37aee4572a71f162bd8db51da55ba4fa/charset_normalizer-3.5.1-cp37-abi3-win_arm64.whl", hash = "sha256:87e4f41d375c0b9be2fb5251aee4b8a689169e134535aed81bf085c3b647451e", size = 287318, upload-time = "2026-08-15T08:20:15.551Z" }, + { url = "https://files.pythonhosted.org/packages/cc/61/d01fc49b8dea277640b55a9e15960dbca9fdc8c9fde18e572d39c59f4019/charset_normalizer-3.5.1-py3-none-any.whl", hash = "sha256:6df0ec430f9a831772c23ca5a224cba36517a58a84bb32c32bb59a9fa67c47f6", size = 68658, upload-time = "2026-08-15T08:20:43.306Z" }, +] + [[package]] name = "click" version = "8.4.2" @@ -361,6 +492,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/03/10388a42375ee7e4ac9b94eb2c5c569c8b5795e377e701c9ac3ad63de890/fastapi-0.141.1-py3-none-any.whl", hash = "sha256:bfb91aa2d334c61cb35ba9a116fc123b3d3df31640b801cf57a7a78ec3f603b3", size = 131954, upload-time = "2026-07-29T17:18:04.364Z" }, ] +[[package]] +name = "googleapis-common-protos" +version = "1.75.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/72/73/74bcab964c9a7a61f2bb71e8179b0f13e6fa98f7ce00fd168aab291e4a2e/googleapis_common_protos-1.75.1.tar.gz", hash = "sha256:d3042c6c5a2d4e67113104d6b6818b59b6bd92a197f2a91508e801fe815cf071", size = 150967, upload-time = "2026-08-06T06:24:51.972Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/51/186c02b8549b69ccda44429cf6ff5081e4b61a602ddfe6a8020d1be31d1b/googleapis_common_protos-1.75.1-py3-none-any.whl", hash = "sha256:28a1934bcd33b9c9da66ac301a0a4227e3367f095a17d0375cb98f0a09d93b79", size = 300626, upload-time = "2026-08-06T06:23:46.696Z" }, +] + [[package]] name = "h11" version = "0.16.0" @@ -459,6 +602,9 @@ source = { editable = "." } dependencies = [ { name = "certifi" }, { name = "cryptography" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-http" }, + { name = "opentelemetry-sdk" }, { name = "pillow" }, { name = "rankweave" }, { name = "rdflib" }, @@ -491,6 +637,9 @@ requires-dist = [ { name = "fast-mlsirm", marker = "extra == 'backend'", git = "https://github.com/ContextualWisdomLab/fast-mlsirm.git?rev=5006c38286a4fa1d81bcf57eeed5ce27ae743f50" }, { name = "fastapi", marker = "extra == 'backend'", specifier = ">=0.115.0" }, { name = "httpx", marker = "extra == 'dev'", specifier = ">=0.27.0" }, + { name = "opentelemetry-api", specifier = ">=1.30.0" }, + { name = "opentelemetry-exporter-otlp-proto-http", specifier = ">=1.30.0" }, + { name = "opentelemetry-sdk", specifier = ">=1.30.0" }, { name = "pillow", specifier = ">=12.3.0" }, { name = "psycopg2-binary", marker = "extra == 'dev'", specifier = ">=2.9.12" }, { name = "pyjwt", extras = ["crypto"], marker = "extra == 'backend'", specifier = ">=2.8.0" }, @@ -577,6 +726,87 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b4/07/458c344f0f0c178f4481dad5cca790626ffe4c34eabf9467069d06ee4999/numpy-2.5.2-cp315-cp315t-win_arm64.whl", hash = "sha256:5f8e00be2ec6f45f4e8a41a527f68d44a7d96fee92a650e4d8b1326f77f61e6e", size = 10748103, upload-time = "2026-08-09T13:48:24.21Z" }, ] +[[package]] +name = "opentelemetry-api" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ee/8b/aa9e2d8b8dfa7c946f7dec5d1f8f6ba8eca062f43509a06bdb5ce93d26c0/opentelemetry_api-1.44.0.tar.gz", hash = "sha256:67647e5e9566edcf421166fdf022b3537f818635daa852b289e34604dc6fb33a", size = 72406, upload-time = "2026-07-16T15:25:32.678Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/6f/a04e900f465ff3221ccc395522503e2d10e79fa21f2723c8e177aae1e0d1/opentelemetry_api-1.44.0-py3-none-any.whl", hash = "sha256:94b98c893a91b88657eaac1e3ba89618cdb85be6918196705354f34728b2cdef", size = 60018, upload-time = "2026-07-16T15:25:11.657Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-common" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-proto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/09/4d717852c1cf3f854b76c7110a5d00883bc3c99288b9b0dbcbeb9e306eb6/opentelemetry_exporter_otlp_proto_common-1.44.0.tar.gz", hash = "sha256:dc87a5a5bc58f149a56d1547e4691588fa12994cdc3bc039a694ccb3375862ac", size = 20202, upload-time = "2026-07-16T15:25:37.658Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/71/65fd9d54c10b860f87c045ccee1264cab7011268895d3528818a29c1172a/opentelemetry_exporter_otlp_proto_common-1.44.0-py3-none-any.whl", hash = "sha256:9a9fe61bba73d802904bc989f1d6b4a7b1ee40f06c40e98d6f85af65aaebb694", size = 17045, upload-time = "2026-07-16T15:25:18.201Z" }, +] + +[[package]] +name = "opentelemetry-exporter-otlp-proto-http" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "googleapis-common-protos" }, + { name = "opentelemetry-api" }, + { name = "opentelemetry-exporter-otlp-proto-common" }, + { name = "opentelemetry-proto" }, + { name = "opentelemetry-sdk" }, + { name = "requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1a/87/95e2a5aaa795b4e2260d74e16df2d5541deb2ea9de010bcd615f4dee2654/opentelemetry_exporter_otlp_proto_http-1.44.0.tar.gz", hash = "sha256:c633d7270ad6b57cd4cfbe8b0007a9e2e7c0cb50bd6c50fe2a7b245f721a09d8", size = 25806, upload-time = "2026-07-16T15:25:39.162Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cd/d0/fdeb1a98d8d3a6205f5f297c51b4a9bfe65126ab60339669bbe3dd54c2e2/opentelemetry_exporter_otlp_proto_http-1.44.0-py3-none-any.whl", hash = "sha256:838592fce774c1c8bb7b9a0a7facbfa82e17be5a8a4e94cef10cb84ae026bae3", size = 21850, upload-time = "2026-07-16T15:25:20.006Z" }, +] + +[[package]] +name = "opentelemetry-proto" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/64/01/40ac4ae9a149263cc52c2cee200ddd80cb6d8db1a4610abf8eabce0fe771/opentelemetry_proto-1.44.0.tar.gz", hash = "sha256:c547a79c2f8c0c515d31509154682e5921c7cfd5ca67b70e1f9266e2c3e103f3", size = 46488, upload-time = "2026-07-16T15:25:45.34Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/7c/8be563d68e93bbefa5c8affb82ddcff91b3ad858ce49957ba7b16fd3e0ab/opentelemetry_proto-1.44.0-py3-none-any.whl", hash = "sha256:898b155a0e1557afd867478fb6158e8122a46329ca0bb8dc53cc55e98f017f56", size = 72483, upload-time = "2026-07-16T15:25:28.429Z" }, +] + +[[package]] +name = "opentelemetry-sdk" +version = "1.44.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "opentelemetry-semantic-conventions" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5d/77/a6592cbc7c8d9bcc9d6757a9df45e04a7c585e3e6e7a13456da522b21109/opentelemetry_sdk-1.44.0.tar.gz", hash = "sha256:cebe7f65dc12f26ead75c6064de12fd2a9052e5060c0272d402cfa203aae123b", size = 208624, upload-time = "2026-07-16T15:25:46.078Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/23/ff077e61886ee020a17ce9c8b6fa11c601c8d8345b09ea24f605445df62a/opentelemetry_sdk-1.44.0-py3-none-any.whl", hash = "sha256:df081c4c6bcfdb1211e3e86140376792643128a25f8d72d1d27675936e7e96ad", size = 137221, upload-time = "2026-07-16T15:25:29.534Z" }, +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.65b0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "opentelemetry-api" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8f/73/0cbdebcb4cf545fdd328da14f5137e37d0770c3f26185e478b0d15d94f50/opentelemetry_semantic_conventions-0.65b0.tar.gz", hash = "sha256:f9b2b81e9d5b64f11bc952075e7e9c7fb0aab075c7fd1c46d597f1b919852d60", size = 148774, upload-time = "2026-07-16T15:25:46.902Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a6/0e/49df70d9b81fb5cbae4bbf2a49d865b09bcbcbc4eb53f5851b1027738d78/opentelemetry_semantic_conventions-0.65b0-py3-none-any.whl", hash = "sha256:1cacde7b0ad306f84c5ef08c3dbe1bbaf20165bba6f8bff43b670e555a086bcb", size = 204645, upload-time = "2026-07-16T15:25:30.688Z" }, +] + [[package]] name = "packaging" version = "26.3" @@ -666,6 +896,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "protobuf" +version = "7.36.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a7/e7/0553e21d25ca4d9f573135775348a372c3ec34a93a71d5f297c3bac38341/protobuf-7.36.0.tar.gz", hash = "sha256:e8e09cb0d794c6687926fa558a8a6e72aa10edb997d5ca61da0765f12a3e00ea", size = 510034, upload-time = "2026-08-20T16:34:01.071Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8f/ae/58e3ca96cb2e118cc546b677359b3c6659f79a140935c08dec94c7998585/protobuf-7.36.0-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:9103532dffd80c6fab7e50c65a31007680a06eb57537d437bb1b35812c138a37", size = 453256, upload-time = "2026-08-20T16:33:53.945Z" }, + { url = "https://files.pythonhosted.org/packages/f0/15/5162230af4912697f0fe406f6800f80760945babcff0e2c2fe6c84ef2d5d/protobuf-7.36.0-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:bf94a5917c71058262de683669bc0a797a7669d3de71f0b36d058e3194f47b44", size = 341436, upload-time = "2026-08-20T16:33:55.134Z" }, + { url = "https://files.pythonhosted.org/packages/d7/09/1670b2bfc9a45e807e520c3e9be36524db9ccc7dc05ea17af7681cabdc61/protobuf-7.36.0-cp310-abi3-manylinux2014_s390x.whl", hash = "sha256:3297e60abdff301e5f74393d87f6cc59dacab5f024a89548a6e8de1d26576b16", size = 354440, upload-time = "2026-08-20T16:33:56.077Z" }, + { url = "https://files.pythonhosted.org/packages/c7/f8/bd5804695ba400e423c33fd4d9f58c28d86633d5ba1945c36ff3967d98cb/protobuf-7.36.0-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:70f5ec8eb0da81a44360c0dc0beac99a0d78071d21956a7076bae8bd2051841b", size = 340439, upload-time = "2026-08-20T16:33:56.992Z" }, + { url = "https://files.pythonhosted.org/packages/ef/9f/acd02338235a3e7d03168c4303478347b7624fc8189ff4e7f0d2654bbe86/protobuf-7.36.0-cp310-abi3-win32.whl", hash = "sha256:7326fd717bdc419162a735938d89d4032332bcc3408804012b24ff3a37086071", size = 440216, upload-time = "2026-08-20T16:33:57.99Z" }, + { url = "https://files.pythonhosted.org/packages/0e/4e/12cb93270967a2affff5b3f720694700d4d87712a67afd05c8cb3f6fa52c/protobuf-7.36.0-cp310-abi3-win_amd64.whl", hash = "sha256:1781cc1de61249b750848029bca452c0a8b7e990080316b9bbc2518b2117b488", size = 453731, upload-time = "2026-08-20T16:33:58.951Z" }, + { url = "https://files.pythonhosted.org/packages/01/c3/629999e78d46c1115c11886d51c6bd68c17ce4a944f1ea3e153a91316a33/protobuf-7.36.0-py3-none-any.whl", hash = "sha256:53374d53fc29a67f7dbbf0ade47d7526a0f0137bf0f9c90e48d8a60790ef748c", size = 177024, upload-time = "2026-08-20T16:34:00.053Z" }, +] + [[package]] name = "psycopg2-binary" version = "2.9.12" @@ -935,6 +1180,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/66/9d/c5731f6e3608663d4d3656fd8d3aecee8b509c3082818f5a13eae925baea/redis-8.1.0-py3-none-any.whl", hash = "sha256:a4fe1aac3d3b3cc791d4b3d5931c5a956045dc951ee74d1c913ee3ac4d2ee9fb", size = 560618, upload-time = "2026-07-30T08:50:58.497Z" }, ] +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + [[package]] name = "starlette" version = "1.6.0" @@ -978,6 +1238,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147", size = 14750, upload-time = "2026-08-12T12:37:24.648Z" }, ] +[[package]] +name = "urllib3" +version = "2.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/0c/06f8b233b8fd13b9e5ee11424ef85419ba0d8ba0b3138bf360be2ff56953/urllib3-2.7.0.tar.gz", hash = "sha256:231e0ec3b63ceb14667c67be60f2f2c40a518cb38b03af60abc813da26505f4c", size = 433602, upload-time = "2026-05-07T16:13:18.596Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, +] + [[package]] name = "uvicorn" version = "0.52.1"