diff --git a/tests/eval/consolidation/test_consolidation_eval.py b/tests/eval/consolidation/test_consolidation_eval.py index 8b4a88d7..c9f85a23 100644 --- a/tests/eval/consolidation/test_consolidation_eval.py +++ b/tests/eval/consolidation/test_consolidation_eval.py @@ -21,6 +21,7 @@ RejectNewDecision, UnifyDecision, ) +from reflexio.test_support.llm_mock import assert_litellm_unpatched from reflexio.test_support.skip_decorators import skip_low_priority from tests.eval.consolidation.case import ( ConsolidationEvalCase, @@ -519,6 +520,10 @@ def test_live_consolidation_provider_real(tmp_path): # pragma: no cover - manua the fixture. Asserts only pipeline mechanics (every case produced one of the four kinds), never exact kinds. Run manually with API keys + RUN_LOW_PRIORITY=1.""" + # This test lives outside ``tests/e2e_tests/``, so the session-wide + # ``litellm.completion`` patch is installed for it. Without this it would + # grade canned mock text while reporting a real model. + assert_litellm_unpatched() from reflexio.server.api_endpoints.request_context import RequestContext from reflexio.server.llm.litellm_client import LiteLLMClient, LiteLLMConfig @@ -544,6 +549,10 @@ def test_live_consolidation_provider_real(tmp_path): # pragma: no cover - manua @skip_low_priority def test_real_judge_smoke(): # pragma: no cover - manual, costs money """Smoke test against a real judge model. Run manually with API keys.""" + # This test lives outside ``tests/e2e_tests/``, so the session-wide + # ``litellm.completion`` patch is installed for it. Without this it would + # grade canned mock text while reporting a real model. + assert_litellm_unpatched() from reflexio.server.llm.litellm_client import LiteLLMClient, LiteLLMConfig client = LiteLLMClient(LiteLLMConfig(model="claude-haiku-4-5")) diff --git a/tests/eval/extraction/test_extraction_eval.py b/tests/eval/extraction/test_extraction_eval.py index 5a6481fc..7c1a8df3 100644 --- a/tests/eval/extraction/test_extraction_eval.py +++ b/tests/eval/extraction/test_extraction_eval.py @@ -14,6 +14,7 @@ import pytest from pydantic import BaseModel +from reflexio.test_support.llm_mock import assert_litellm_unpatched from reflexio.test_support.skip_decorators import skip_low_priority from tests.eval.conftest import _load, _load_rubric from tests.eval.extraction.providers import make_extraction_provider @@ -180,6 +181,10 @@ def test_score_golden_case(extraction_case, extraction_judge): @skip_low_priority def test_real_judge_smoke(): # pragma: no cover - manual, costs money """Smoke test against a real judge model. Run manually with API keys.""" + # This test lives outside ``tests/e2e_tests/``, so the session-wide + # ``litellm.completion`` patch is installed for it. Without this it would + # grade canned mock text while reporting a real model. + assert_litellm_unpatched() from reflexio.server.llm.litellm_client import LiteLLMClient, LiteLLMConfig rubric = _load_rubric("extraction_rubric.yaml") @@ -298,6 +303,10 @@ def test_live_extraction_provider_real(tmp_path): # pragma: no cover - manual in range), never exact scores. Run manually with API keys + RUN_LOW_PRIORITY=1. """ + # This test lives outside ``tests/e2e_tests/``, so the session-wide + # ``litellm.completion`` patch is installed for it. Without this it would + # grade canned mock text while reporting a real model. + assert_litellm_unpatched() from reflexio.server.api_endpoints.request_context import RequestContext from reflexio.server.llm.litellm_client import LiteLLMClient, LiteLLMConfig diff --git a/tests/eval/scenarios/test_scenario_eval.py b/tests/eval/scenarios/test_scenario_eval.py index 9cf4e85a..93e11f62 100644 --- a/tests/eval/scenarios/test_scenario_eval.py +++ b/tests/eval/scenarios/test_scenario_eval.py @@ -14,6 +14,7 @@ RejectNewDecision, UnifyDecision, ) +from reflexio.test_support.llm_mock import assert_litellm_unpatched from reflexio.test_support.skip_decorators import skip_low_priority from tests.eval.consolidation.judge import ConsolidationVerdict from tests.eval.scenarios.book import _next_id, apply_consolidation @@ -172,6 +173,10 @@ def test_wrong_consolidation_verdict_fails_scenario() -> None: @skip_low_priority def test_scenario_real(tmp_path) -> None: # pragma: no cover - manual, costs money + # This test lives outside ``tests/e2e_tests/``, so the session-wide + # ``litellm.completion`` patch is installed for it. Without this it would + # grade canned mock text while reporting a real model. + assert_litellm_unpatched() from reflexio.server.api_endpoints.request_context import RequestContext from reflexio.server.llm.litellm_client import LiteLLMClient, LiteLLMConfig from tests.eval.consolidation.providers import (