diff --git a/CHANGELOG.md b/CHANGELOG.md index 0096828a2..324e3f5bd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,18 @@ All notable changes to this project are documented here. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versioning follows [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Changed + +- Active contextual-orchestrator clients now request `mode="auto"` rather than forcing a one-model route. The orchestrator owns the minimum-cost route, verification, or conducted workflow that satisfies the detected quality requirement; explicit modes remain available for controlled experiments and operator overrides. + +## [Unreleased] + +### Changed + +- Product LLM adapters now delegate their default execution tier to contextual-orchestrator `auto` instead of forcing a single-model `route`; the explicit adjudication `verify` contract remains unchanged. + ## [0.71.0] - 2026-08-14 ### Added diff --git a/docs/adr/0005-adaptive-contextual-orchestrator-default.md b/docs/adr/0005-adaptive-contextual-orchestrator-default.md new file mode 100644 index 000000000..790046e58 --- /dev/null +++ b/docs/adr/0005-adaptive-contextual-orchestrator-default.md @@ -0,0 +1,37 @@ +# ADR-0005: Product LLM clients delegate default execution to contextual-orchestrator auto + +- Status: Accepted +- Date: 2026-08-16 + +## Context + +LineageWeave had several independent feature adapters for summarization, Keyman +extraction, relationship classification, commitments, chat, and post evaluation. +Each adapter hard-coded `route`, which duplicated policy and forced a single worker +regardless of uncertainty, risk, or task complexity. Adjudication separately uses +`verify` because it is an explicit controlled worker-plus-checker contract. + +## Decision + +Every production adapter that does not intentionally implement a controlled +ablation sends `mode="auto"`. Contextual-orchestrator owns model/provider selection, +reasoning effort, verification depth, failover, and the quality-first/cost-aware +execution tier. The explicit adjudication `verify` contract remains unchanged. + +The application still owns prompt semantics, strict parsers, typed domain records, +tenant authorization, persistence, and fail-closed handling. Auto orchestration is +not permission to accept malformed or unsupported model output. + +## Consequences + +A simple extraction may still resolve to one worker when that is the +quality-sufficient least-cost plan. Evaluation and uncertain classification can use +a verifier, while complex synthesis can use a conducted workflow, without changing +LineageWeave's public interfaces. Returned trace and usage evidence remain available +for empirical calibration. + +## References + +Omidvar, H., & Akhlaghi, V. (2026). *A communication-theoretic framework for LLM agents: Cost-aware adaptive reliability* [Preprint]. arXiv. https://doi.org/10.48550/arXiv.2605.09121 + +Tang, Y., Cetin, E., Xu, J., Sun, Q., Nielsen, S., Richard, V., Goda, H., Tymchenko, I., Nguyen, N., Lee, H., Ashiga, M., Kotyan, S., Kuroki, S., & Clanuwat, T. (2026). *Sakana Fugu technical report* [Technical report]. arXiv. https://doi.org/10.48550/arXiv.2606.21228 diff --git a/docs/adr/0005-adaptive-orchestrator-default.md b/docs/adr/0005-adaptive-orchestrator-default.md new file mode 100644 index 000000000..c64585f8d --- /dev/null +++ b/docs/adr/0005-adaptive-orchestrator-default.md @@ -0,0 +1,45 @@ +# ADR-0005: Adaptive contextual-orchestrator mode is the consumer default + +- Status: Accepted +- Date: 2026-08-15 + +## Context + +LineageWeave previously forced `mode="route"` in summarization, post evaluation, +Keyman extraction, commitment extraction, post chat, and relationship +classification. That made the consumer choose a single model before +contextual-orchestrator could evaluate task difficulty, capability fit, +verification need, and known model price. + +Research on adaptive orchestration and cost-aware reliability shows that no fixed +model/workflow/budget choice dominates for all requests. Dynamic scaffolding and +query-level cost allocation are therefore responsibilities of the orchestration +plane, not of each domain client. + +## Decision + +Active general-purpose clients request `mode="auto"`. + +- contextual-orchestrator selects the quality-sufficient route, bounded + verification, or conducted workflow and then minimizes known cost inside the + selected capability tier; +- LineageWeave continues to own prompts, schemas, strict parsing, domain evidence, + and failure semantics; +- the low-volume lineage adjudication channel retains the explicit `verify` + override because an independently checked verdict is part of that domain + contract, not an accidental routing default; +- explicit modes remain permitted for ablation, regression comparison, and + emergency operator policy, but they are not ordinary production defaults. + +## Consequences + +Trace width is no longer a stable consumer assumption for `auto` requests. +Telemetry and tests must record the requested policy and actual trace. Cost +claims require configured price evidence; an unpriced model is never treated as +free. + +## References + +Omidvar, H., & Akhlaghi, V. (2026). *A communication-theoretic framework for LLM agents: Cost-aware adaptive reliability* [Preprint]. arXiv. https://doi.org/10.48550/arXiv.2605.09121 + +Tang, Y., Cetin, E., Xu, J., Sun, Q., Nielsen, S., Richard, V., Goda, H., Tymchenko, I., Nguyen, N., Lee, H., Ashiga, M., Kotyan, S., Kuroki, S., & Clanuwat, T. (2026). *Sakana Fugu technical report* [Technical report]. arXiv. https://doi.org/10.48550/arXiv.2606.21228 diff --git a/lineageweave/commitment_extraction.py b/lineageweave/commitment_extraction.py index 341eaa0c8..459376617 100644 --- a/lineageweave/commitment_extraction.py +++ b/lineageweave/commitment_extraction.py @@ -136,7 +136,7 @@ def parse_commitment_response(content: str) -> CustomerCommitment | None: class ContextualOrchestratorCommitmentExtractionClient: - """Calls ``POST {base_url}/v1/chat/completions`` with ``mode="route"``.""" + """Calls ``POST {base_url}/v1/chat/completions`` with ``mode="auto"``.""" available = True @@ -156,7 +156,7 @@ def extract(self, post_title: str, post_body: str, reference_date: str) -> Custo f"{self._base_url}/v1/chat/completions", { "messages": [{"role": "user", "content": prompt}], - "mode": "route", + "mode": "auto", "reasoning_effort": self._reasoning_effort, }, headers={"authorization": f"Bearer {self._api_key}"}, diff --git a/lineageweave/entity_relationship_classification.py b/lineageweave/entity_relationship_classification.py index d1a2056ca..ad23e7d9a 100644 --- a/lineageweave/entity_relationship_classification.py +++ b/lineageweave/entity_relationship_classification.py @@ -157,7 +157,7 @@ def parse_classification_response( class ContextualOrchestratorEntityRelationshipClient: - """Calls ``POST {base_url}/v1/chat/completions`` with ``mode="route"``.""" + """Calls ``POST {base_url}/v1/chat/completions`` with ``mode="auto"``.""" available = True @@ -183,7 +183,7 @@ def classify( f"{self._base_url}/v1/chat/completions", { "messages": [{"role": "user", "content": prompt}], - "mode": "route", + "mode": "auto", "reasoning_effort": self._reasoning_effort, }, headers={"authorization": f"Bearer {self._api_key}"}, diff --git a/lineageweave/keyman_extraction.py b/lineageweave/keyman_extraction.py index d7e8de952..d202de9bd 100644 --- a/lineageweave/keyman_extraction.py +++ b/lineageweave/keyman_extraction.py @@ -10,11 +10,11 @@ :class:`ContextualOrchestratorKeymanExtractionClient` calls a running contextual-orchestrator instance -- never a raw LLM API directly, per AGENTS.md -- because Keyman identification is a structured-extraction task -that benefits from the orchestrator's reasoning-effort allocation, not a -single confidence number, so it uses ``mode="route"`` (one worker call) at -a ``"medium"`` reasoning effort by default rather than ``verify``'s -worker-plus-checker pattern, which is reserved for adjudication's binary -judgment calls. +that benefits from the orchestrator's task-sensitive allocation of model, +reasoning effort, and workflow depth. It therefore uses ``mode="auto"`` at +a ``"medium"`` reasoning effort by default; contextual-orchestrator may use +a single worker or escalate to verification/conducted work when the detected +quality requirement justifies the additional cost. """ from __future__ import annotations @@ -133,7 +133,7 @@ def parse_keyman_response(content: str) -> list[PersonMention]: class ContextualOrchestratorKeymanExtractionClient: - """Calls ``POST {base_url}/v1/chat/completions`` with ``mode="route"``.""" + """Calls ``POST {base_url}/v1/chat/completions`` with ``mode="auto"``.""" available = True @@ -151,7 +151,7 @@ def extract(self, post_title: str, post_body: str) -> list[PersonMention]: f"{self._base_url}/v1/chat/completions", { "messages": [{"role": "user", "content": prompt}], - "mode": "route", + "mode": "auto", "reasoning_effort": self._reasoning_effort, }, headers={"authorization": f"Bearer {self._api_key}"}, diff --git a/lineageweave/post_chat.py b/lineageweave/post_chat.py index 7ec67e943..d23624bc1 100644 --- a/lineageweave/post_chat.py +++ b/lineageweave/post_chat.py @@ -187,7 +187,7 @@ class ContextualOrchestratorPostChatClient: ``mode="verify"`` exists for (one worker call plus one checked verifier judgment), same reasoning ``adjudication_client`` already uses, not ``keyman_extraction``/``entity_relationship_classification``'s - single-pass ``mode="route"`` structured extraction. + single-pass ``mode="auto"`` structured extraction. """ available = True diff --git a/lineageweave/post_evaluation.py b/lineageweave/post_evaluation.py index 09ce21fd2..2d3be6291 100644 --- a/lineageweave/post_evaluation.py +++ b/lineageweave/post_evaluation.py @@ -85,7 +85,7 @@ def __init__(self, base_url: str, api_key: str, *, timeout: float = 60.0) -> Non self._api_key = api_key self._timeout = timeout - def complete(self, messages: list[dict[str, Any]], mode: str = "route") -> dict[str, Any]: + def complete(self, messages: list[dict[str, Any]], mode: str = "auto") -> dict[str, Any]: body = post_json( f"{self._base_url}/v1/chat/completions", {"messages": messages, "mode": mode, "reasoning_effort": "medium"}, @@ -107,7 +107,7 @@ class ContextualOrchestratorPostEvaluationClient: def __init__(self, base_url: str, api_key: str, *, timeout: float = 60.0) -> None: self._judge = ContextualOrchestratorJudge( _OrchestratorCompleteAdapter(base_url, api_key, timeout=timeout), - mode="route", + mode="auto", ) def evaluate(self, post_title: str, post_body: str) -> LLMJudgeResult: diff --git a/lineageweave/post_summary.py b/lineageweave/post_summary.py index 4974268c2..eac051fda 100644 --- a/lineageweave/post_summary.py +++ b/lineageweave/post_summary.py @@ -152,7 +152,7 @@ def parse_summary_response(content: str) -> PostSummary | None: class ContextualOrchestratorPostSummaryClient: - """Calls ``POST {base_url}/v1/chat/completions`` with ``mode="route"``.""" + """Calls ``POST {base_url}/v1/chat/completions`` with ``mode="auto"``.""" available = True @@ -170,7 +170,7 @@ def summarize(self, post_title: str, post_body: str) -> PostSummary: f"{self._base_url}/v1/chat/completions", { "messages": [{"role": "user", "content": prompt}], - "mode": "route", + "mode": "auto", "reasoning_effort": self._reasoning_effort, }, headers={"authorization": f"Bearer {self._api_key}"}, diff --git a/tests/test_adaptive_orchestrator_default.py b/tests/test_adaptive_orchestrator_default.py new file mode 100644 index 000000000..98affc0c0 --- /dev/null +++ b/tests/test_adaptive_orchestrator_default.py @@ -0,0 +1,55 @@ +"""LineageWeave delegates product-default LLM execution to auto policy.""" + +from __future__ import annotations + +from pathlib import Path + +from lineageweave import post_evaluation + + +class _Response: + """OpenAI-compatible response body used by the transport seam.""" + + choices = [{"message": {"content": "{}"}}] + + +def test_post_evaluation_adapter_defaults_to_auto(monkeypatch) -> None: + observed: dict[str, object] = {} + + def fake_post_json(url, payload, *, headers, timeout): + observed.update( + url=url, + payload=payload, + headers=headers, + timeout=timeout, + ) + return {"choices": [{"message": {"content": "{}"}}]} + + monkeypatch.setattr(post_evaluation, "post_json", fake_post_json) + adapter = post_evaluation._OrchestratorCompleteAdapter( + "https://orchestrator.example.test", "inference_token" + ) + adapter.complete([{"role": "user", "content": "Evaluate this evidence."}]) + + assert observed["payload"]["mode"] == "auto" + + +def test_post_evaluation_judge_uses_auto_by_default() -> None: + client = post_evaluation.ContextualOrchestratorPostEvaluationClient( + "https://orchestrator.example.test", "inference_token" + ) + assert client._judge.mode == "auto" + + +def test_runtime_clients_do_not_force_single_model_route() -> None: + package_root = Path(__file__).resolve().parents[1] / "lineageweave" + violations: list[str] = [] + for path in sorted(package_root.glob("*.py")): + text = path.read_text(encoding="utf-8") + if '"mode": "route"' in text or "'mode': 'route'" in text: + violations.append(f"{path.name}: request payload") + if 'mode="route"' in text or "mode='route'" in text: + violations.append(f"{path.name}: constructor/call default") + if 'mode: str = "route"' in text or "mode: str = 'route'" in text: + violations.append(f"{path.name}: typed default") + assert violations == [] diff --git a/tests/test_contextual_orchestrator_default_policy.py b/tests/test_contextual_orchestrator_default_policy.py new file mode 100644 index 000000000..6c5eb5d7c --- /dev/null +++ b/tests/test_contextual_orchestrator_default_policy.py @@ -0,0 +1,40 @@ +"""Contract tests for adaptive contextual-orchestrator consumer defaults.""" +from __future__ import annotations + +from pathlib import Path +import unittest + +ROOT = Path(__file__).resolve().parents[1] +ACTIVE_CLIENTS = ( + "lineageweave/post_summary.py", + "lineageweave/post_evaluation.py", + "lineageweave/keyman_extraction.py", + "lineageweave/commitment_extraction.py", + "lineageweave/post_chat.py", + "lineageweave/entity_relationship_classification.py", +) + + +class AdaptiveOrchestratorDefaultTest(unittest.TestCase): + """Protect production clients from regressing to forced one-model routing.""" + + def test_active_clients_use_auto_and_never_force_route(self) -> None: + for relative in ACTIVE_CLIENTS: + source = (ROOT / relative).read_text(encoding="utf-8") + with self.subTest(path=relative): + self.assertNotIn('"mode": "route"', source) + self.assertNotIn('mode="route"', source) + self.assertNotIn('mode: str = "route"', source) + self.assertTrue( + '"mode": "auto"' in source + or 'mode="auto"' in source + or 'mode: str = "auto"' in source + ) + + def test_high_stakes_adjudication_retains_explicit_checked_override(self) -> None: + source = (ROOT / "lineageweave/adjudication_client.py").read_text(encoding="utf-8") + self.assertIn('"mode": "verify"', source) + + +if __name__ == "__main__": + unittest.main()