Boost benchmark/reranker/embedder coverage past codecov target

Patch coverage on PR #1380 was 85.02% (target 87.30%). Added:

- run_benchmark management command: full happy-path test (real adapter,
  mocked runner) plus retrieval-only/corpus-wide flag passthrough and
  user-not-found error path. Brings ~50 lines of run_benchmark.py from
  0% to covered.
- CohereReranker fault-tolerance: request-exception, non-200, non-JSON
  body, missing 'results' key, malformed item skipping, and empty
  results all fall back to identity ordering.
- CrossEncoderReranker (no prior tests): success path with score
  extraction, max_length forwarding, padding when the model returns
  fewer scores than passages, scalar→list normalization, and the
  per-key model cache fast path. Uses an injected stub model so CI
  runs without sentence-transformers/torch installed.
- OpenAI embedder truncation: oversize single-text and oversize batch
  inputs are clipped to OPENAI_EMBEDDER_MAX_INPUT_CHARS before the
  wire call; empty/whitespace batch inputs become None slots.

Author: JSv4

opencontractserver/tests/test_benchmarks.py

@@ -842,3 +842,114 @@ def test_sampling_drops_malformed_tests(self):
         self.assertEqual(len(out), 194)
         for t in out:
             self.assertTrue(t["snippets"][0].get("file_path"))
+
+
+# --------------------------------------------------------------------------- #
+# Management command tests (``python manage.py run_benchmark``)
+# --------------------------------------------------------------------------- #
+
+
+class RunBenchmarkCommandTest(TransactionTestCase):
+    """Cover the CLI entry point so user lookup, adapter wiring and
+    aggregate printing are exercised end-to-end with the runner mocked.
+    """
+
+    def test_user_not_found_raises(self):
+        from django.core.management import call_command
+        from django.core.management.base import CommandError
+
+        with self.assertRaisesRegex(CommandError, "not found"):
+            call_command(
+                "run_benchmark",
+                "--path=/tmp/does-not-matter",
+                "--user=nope-no-such-user",
+            )
+
+    @patch(
+        "opencontractserver.benchmarks.management.commands.run_benchmark."
+        "run_benchmark"
+    )
+    def test_happy_path_invokes_runner_and_prints_aggregates(self, mock_run):
+        from io import StringIO
+
+        from django.core.management import call_command
+
+        from opencontractserver.benchmarks.report import BenchmarkReport
+
+        user = User.objects.create_user(username="bench-cli-user")
+
+        # Use the real BenchmarkReport so __post_init__ populates aggregates
+        # — covers the float/dict/value formatting branches in handle().
+        report = BenchmarkReport(
+            adapter={"name": "test"},
+            config={"model": "test:m"},
+            corpus_id=42,
+            extract_id=7,
+            task_results=[],
+            run_dir=Path("/tmp/run-x"),
+        )
+        mock_run.return_value = report
+
+        out = StringIO()
+        call_command(
+            "run_benchmark",
+            f"--path={MICRO_FIXTURE}",
+            "--user=bench-cli-user",
+            "--top-k=3",
+            "--limit=5",
+            stdout=out,
+        )
+
+        # Runner called with the parsed CLI options.
+        run_kwargs = mock_run.call_args.kwargs
+        self.assertEqual(run_kwargs["top_k"], 3)
+        self.assertEqual(run_kwargs["user"], user)
+        self.assertFalse(run_kwargs["retrieval_only"])
+        self.assertFalse(run_kwargs["corpus_wide"])
+        # The adapter passed to run_benchmark is a real LegalBenchRAGAdapter.
+        from opencontractserver.benchmarks.adapters.legalbench_rag import (
+            LegalBenchRAGAdapter,
+        )
+
+        self.assertIsInstance(run_kwargs["adapter"], LegalBenchRAGAdapter)
+
+        text = out.getvalue()
+        self.assertIn("Benchmark run complete", text)
+        self.assertIn("Corpus ID:  42", text)
+        self.assertIn("Extract ID: 7", text)
+        self.assertIn("Report dir: /tmp/run-x", text)
+        # Aggregate lines emitted (BenchmarkReport.__post_init__ populates
+        # task_count and float metrics, exercising the int-else and float
+        # branches of the handle() output loop).
+        self.assertIn("task_count", text)
+        self.assertIn("answer_token_f1", text)
+
+    @patch(
+        "opencontractserver.benchmarks.management.commands.run_benchmark."
+        "run_benchmark"
+    )
+    def test_retrieval_only_and_corpus_wide_flags_pass_through(self, mock_run):
+        from django.core.management import call_command
+
+        from opencontractserver.benchmarks.report import BenchmarkReport
+
+        User.objects.create_user(username="bench-flag-user")
+        mock_run.return_value = BenchmarkReport(
+            adapter={},
+            config={},
+            corpus_id=1,
+            extract_id=1,
+            task_results=[],
+            run_dir=None,  # exercise the no-run-dir branch in handle()
+        )
+        call_command(
+            "run_benchmark",
+            f"--path={MICRO_FIXTURE}",
+            "--user=bench-flag-user",
+            "--retrieval-only",
+            "--corpus-wide",
+            "--no-paper-sampling",
+        )
+        run_kwargs = mock_run.call_args.kwargs
+        self.assertTrue(run_kwargs["retrieval_only"])
+        self.assertTrue(run_kwargs["corpus_wide"])

opencontractserver/tests/test_openai_embedder.py

@@ -299,6 +299,74 @@ def test_embed_image_not_supported(self):
         result = embedder.embed_image("base64data", "jpeg")
         self.assertIsNone(result)
 
+    @patch("opencontractserver.pipeline.embedders.openai_embedder.openai.OpenAI")
+    def test_embed_text_truncates_oversize_input(self, mock_openai_cls):
+        """Inputs longer than OPENAI_EMBEDDER_MAX_INPUT_CHARS are truncated."""
+        from opencontractserver.constants.document_processing import (
+            OPENAI_EMBEDDER_MAX_INPUT_CHARS,
+        )
+
+        fake_embedding = [0.1] * DEFAULT_OPENAI_EMBEDDING_DIMENSIONS
+        mock_client = MagicMock()
+        mock_client.embeddings.create.return_value = self._make_mock_response(
+            fake_embedding
+        )
+        mock_openai_cls.return_value = mock_client
+
+        oversize = "a" * (OPENAI_EMBEDDER_MAX_INPUT_CHARS + 5_000)
+        embedder = OpenAIEmbedder()
+        result = embedder.embed_text(oversize, openai_api_key="test-key")
+
+        self.assertIsNotNone(result)
+        sent = mock_client.embeddings.create.call_args.kwargs["input"]
+        self.assertEqual(len(sent), OPENAI_EMBEDDER_MAX_INPUT_CHARS)
+
+    @patch("opencontractserver.pipeline.embedders.openai_embedder.openai.OpenAI")
+    def test_embed_texts_batch_truncates_and_skips_blanks(self, mock_openai_cls):
+        """Batch path: blanks become None slots, oversize inputs are clipped."""
+        from opencontractserver.constants.document_processing import (
+            OPENAI_EMBEDDER_MAX_INPUT_CHARS,
+        )
+
+        oversize = "z" * (OPENAI_EMBEDDER_MAX_INPUT_CHARS + 1_000)
+        texts = ["hi", "", oversize, "  "]
+        # Two non-empty inputs survive — return one fake embedding for each.
+        fake_a = [0.1] * DEFAULT_OPENAI_EMBEDDING_DIMENSIONS
+        fake_b = [0.2] * DEFAULT_OPENAI_EMBEDDING_DIMENSIONS
+        mock_response = MagicMock()
+        d0, d1 = MagicMock(), MagicMock()
+        d0.embedding = fake_a
+        d1.embedding = fake_b
+        mock_response.data = [d0, d1]
+        mock_client = MagicMock()
+        mock_client.embeddings.create.return_value = mock_response
+        mock_openai_cls.return_value = mock_client
+
+        embedder = OpenAIEmbedder()
+        result = embedder.embed_texts_batch(texts, openai_api_key="test-key")
+
+        self.assertEqual(len(result), 4)
+        self.assertEqual(result[0], fake_a)  # "hi"
+        self.assertIsNone(result[1])  # "" filtered out
+        self.assertEqual(result[2], fake_b)  # oversize → truncated then sent
+        self.assertIsNone(result[3])  # whitespace-only filtered out
+
+        # Confirm the wire payload carried only the two surviving inputs and
+        # that the oversize one was truncated to the cap.
+        sent = mock_client.embeddings.create.call_args.kwargs["input"]
+        self.assertEqual(len(sent), 2)
+        self.assertEqual(sent[0], "hi")
+        self.assertEqual(len(sent[1]), OPENAI_EMBEDDER_MAX_INPUT_CHARS)
+
+    def test_embed_texts_batch_empty_returns_empty_list(self):
+        embedder = OpenAIEmbedder()
+        self.assertEqual(embedder.embed_texts_batch([]), [])
+
+    def test_embed_texts_batch_all_blank_returns_all_none(self):
+        embedder = OpenAIEmbedder()
+        result = embedder.embed_texts_batch(["", "  ", None])  # type: ignore[list-item]
+        self.assertEqual(result, [None, None, None])
+
 
 class TestOpenAIEmbedderDiscovery(TestCase):
     """Tests that OpenAIEmbedder is properly discovered by the registry."""

opencontractserver/tests/test_reranker.py

@@ -298,6 +298,165 @@ def test_missing_api_key_returns_identity(self) -> None:
         results = reranker.rerank("query", ["a", "b"])
         self.assertEqual([r.index for r in results], [0, 1])
 
+    def test_request_exception_falls_back_to_identity(self) -> None:
+        reranker = self._reranker()
+        with patch(
+            "requests.post",
+            side_effect=requests.exceptions.ConnectionError("network down"),
+        ):
+            results = reranker.rerank("query", ["p0", "p1"])
+        self.assertEqual([r.index for r in results], [0, 1])
+
+    def test_non_200_status_falls_back_to_identity(self) -> None:
+        reranker = self._reranker()
+        with patch("requests.post", return_value=_FakeResponse(status_code=503)):
+            results = reranker.rerank("query", ["p0", "p1"])
+        self.assertEqual([r.index for r in results], [0, 1])
+
+    def test_non_json_body_falls_back_to_identity(self) -> None:
+        class _BadJsonResponse:
+            status_code = 200
+            text = "not json"
+
+            def json(self):
+                raise ValueError("bad json")
+
+        reranker = self._reranker()
+        with patch("requests.post", return_value=_BadJsonResponse()):
+            results = reranker.rerank("query", ["p0", "p1"])
+        self.assertEqual([r.index for r in results], [0, 1])
+
+    def test_missing_results_key_falls_back_to_identity(self) -> None:
+        reranker = self._reranker()
+        with patch(
+            "requests.post",
+            return_value=_FakeResponse(status_code=200, payload={"nope": []}),
+        ):
+            results = reranker.rerank("query", ["p0", "p1"])
+        self.assertEqual([r.index for r in results], [0, 1])
+
+    def test_malformed_items_are_skipped(self) -> None:
+        reranker = self._reranker()
+        fake = _FakeResponse(
+            status_code=200,
+            payload={
+                "results": [
+                    {"index": 0, "relevance_score": "not-a-float"},
+                    {"index": "bad", "relevance_score": 0.9},
+                    {"missing_keys": True},
+                    {"index": 1, "relevance_score": 0.42},
+                ]
+            },
+        )
+        with patch("requests.post", return_value=fake):
+            results = reranker.rerank("query", ["p0", "p1"])
+        # Only the well-formed item survives.
+        self.assertEqual([r.index for r in results], [1])
+
+    def test_empty_results_falls_back_to_identity(self) -> None:
+        reranker = self._reranker()
+        fake = _FakeResponse(status_code=200, payload={"results": []})
+        with patch("requests.post", return_value=fake):
+            results = reranker.rerank("query", ["p0", "p1"])
+        self.assertEqual([r.index for r in results], [0, 1])
+
+
+# --------------------------------------------------------------------------- #
+# Cross-encoder reranker tests (uses an injected stub model so no
+# ``sentence-transformers`` / ``torch`` install is required in CI).
+# --------------------------------------------------------------------------- #
+
+
+class _StubCrossEncoder:
+    """Stand-in for sentence_transformers.CrossEncoder used in tests."""
+
+    def __init__(self, scores):
+        self._scores = scores
+        self.max_length: int | None = None
+        self.last_pairs: list = []
+
+    def predict(self, pairs, batch_size=None, show_progress_bar=False):
+        self.last_pairs = list(pairs)
+        return self._scores
+
+
+class CrossEncoderRerankerTest(TestCase):
+    def setUp(self) -> None:
+        from opencontractserver.pipeline.rerankers import cross_encoder_reranker
+
+        # Reset the module-level cache between tests so each gets a fresh
+        # stub model.
+        cross_encoder_reranker._MODEL_CACHE.clear()
+
+    def _reranker(self):
+        from opencontractserver.pipeline.rerankers.cross_encoder_reranker import (
+            CrossEncoderReranker,
+        )
+
+        rk = CrossEncoderReranker()
+        rk._settings = CrossEncoderReranker.Settings(
+            model_name="stub-model",
+            device="cpu",
+            batch_size=8,
+            max_length=128,
+        )
+        return rk
+
+    def test_successful_rerank_returns_scores(self) -> None:
+        stub = _StubCrossEncoder([0.1, 0.9, 0.5])
+        with patch(
+            "opencontractserver.pipeline.rerankers.cross_encoder_reranker."
+            "_load_cross_encoder",
+            return_value=stub,
+        ):
+            results = self._reranker().rerank("q", ["a", "b", "c"])
+        scores = {r.index: r.score for r in results}
+        self.assertAlmostEqual(scores[0], 0.1)
+        self.assertAlmostEqual(scores[1], 0.9)
+        self.assertAlmostEqual(scores[2], 0.5)
+        # max_length is forwarded onto the underlying model.
+        self.assertEqual(stub.max_length, 128)
+        # Empty/None passages are normalized to "" before pairing.
+        self.assertEqual(stub.last_pairs[0], ("q", "a"))
+
+    def test_pads_when_model_returns_too_few_scores(self) -> None:
+        stub = _StubCrossEncoder([0.7])  # only 1 score for 3 passages
+        with patch(
+            "opencontractserver.pipeline.rerankers.cross_encoder_reranker."
+            "_load_cross_encoder",
+            return_value=stub,
+        ):
+            results = self._reranker().rerank("q", ["a", "b", "c"])
+        # First passage keeps the real score; the rest get -inf padding.
+        self.assertAlmostEqual(results[0].score, 0.7)
+        self.assertEqual(results[1].score, float("-inf"))
+        self.assertEqual(results[2].score, float("-inf"))
+
+    def test_scalar_score_is_normalized_to_list(self) -> None:
+        # Some single-pair responses come back as a 0-D scalar instead of a
+        # sequence; the reranker normalizes by wrapping it.
+        stub = _StubCrossEncoder(0.42)
+        with patch(
+            "opencontractserver.pipeline.rerankers.cross_encoder_reranker."
+            "_load_cross_encoder",
+            return_value=stub,
+        ):
+            results = self._reranker().rerank("q", ["only"])
+        self.assertEqual(len(results), 1)
+        self.assertAlmostEqual(results[0].score, 0.42)
+
+    def test_load_cross_encoder_caches_model_per_key(self) -> None:
+        from opencontractserver.pipeline.rerankers import cross_encoder_reranker
+
+        sentinel = object()
+        with patch(
+            "opencontractserver.pipeline.rerankers.cross_encoder_reranker."
+            "_MODEL_CACHE",
+            {("model-a", "cpu"): sentinel},
+        ):
+            got = cross_encoder_reranker._load_cross_encoder("model-a", "cpu")
+        self.assertIs(got, sentinel)
+
 
 # --------------------------------------------------------------------------- #
 # Pipeline utility tests