From cf3d82fe3fb34d3e6c143e96189a46a8f7550ae7 Mon Sep 17 00:00:00 2001 From: nuwangeek Date: Thu, 25 Jun 2026 17:17:37 +0530 Subject: [PATCH 1/5] Fixed deepeval tests for api tool calling --- .github/workflows/deepeval-tests.yml | 67 +- .../api_tool_report_generator.py | 257 ++++++ tests/deepeval_tests/api_tool_tests.py | 779 ++++++++++++++++++ tests/deepeval_tests/conftest.py | 117 ++- 4 files changed, 1207 insertions(+), 13 deletions(-) create mode 100644 tests/deepeval_tests/api_tool_report_generator.py create mode 100644 tests/deepeval_tests/api_tool_tests.py diff --git a/.github/workflows/deepeval-tests.yml b/.github/workflows/deepeval-tests.yml index cbfe0f0..5ed338a 100644 --- a/.github/workflows/deepeval-tests.yml +++ b/.github/workflows/deepeval-tests.yml @@ -209,8 +209,13 @@ jobs: ENCRYPTION_KEY: ${{ secrets.ENCRYPTION_KEY }} SALT: ${{ secrets.SALT }} run: | - # Run tests sequentially (one at a time) to avoid rate limiting - uv run python -m pytest tests/deepeval_tests/standard_tests.py -v --tb=short --log-cli-level=INFO -n 0 + # Run tests sequentially (one at a time) to avoid rate limiting. + # standard_tests.py — RAG quality (DeepEval metrics on /orchestrate-eval) + # api_tool_tests.py — API Tool Calling scenarios (issue #447) + uv run python -m pytest \ + tests/deepeval_tests/standard_tests.py \ + tests/deepeval_tests/api_tool_tests.py \ + -v --tb=short --log-cli-level=INFO -n 0 - name: Fix permissions on test artifacts if: always() @@ -221,7 +226,11 @@ jobs: - name: Generate evaluation report if: always() run: uv run python tests/deepeval_tests/report_generator.py - + + - name: Generate API tool evaluation report + if: always() + run: uv run python tests/deepeval_tests/api_tool_report_generator.py + - name: Save test artifacts if: always() uses: actions/upload-artifact@v4 @@ -230,8 +239,10 @@ jobs: path: | pytest_captured_results.json test_report.md + api_tool_test_results.json + api_tool_test_report.md retention-days: 30 - + - name: Comment PR with test results if: always() && github.event_name == 'pull_request' uses: actions/github-script@v7 @@ -245,12 +256,12 @@ jobs: repo: context.repo.repo, issue_number: context.issue.number }); - + const existingComment = comments.data.find( comment => comment.user.login === 'github-actions[bot]' && comment.body.includes('RAG System Evaluation Report') ); - + if (existingComment) { await github.rest.issues.updateComment({ owner: context.repo.owner, @@ -275,6 +286,50 @@ jobs: body: `## RAG System Evaluation Report\n\n**Error generating test report**\n\nFailed to read or post test results. Check workflow logs for details.\n\nError: ${error.message}` }); } + + - name: Comment PR with API tool test results + if: always() && github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + try { + const reportContent = fs.readFileSync('api_tool_test_report.md', 'utf8'); + const comments = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number + }); + + const existingComment = comments.data.find( + comment => comment.user.login === 'github-actions[bot]' && + comment.body.includes('API Tool Calling Evaluation Report') + ); + + if (existingComment) { + await github.rest.issues.updateComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: existingComment.id, + body: reportContent + }); + } else { + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + body: reportContent + }); + } + } catch (error) { + console.error('Failed to post API tool test results:', error); + await github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `## API Tool Calling Evaluation Report\n\n**Error generating report**\n\nFailed to read or post API tool test results. Check workflow logs.\n\nError: ${error.message}` + }); + } - name: Check test results and fail if needed if: always() diff --git a/tests/deepeval_tests/api_tool_report_generator.py b/tests/deepeval_tests/api_tool_report_generator.py new file mode 100644 index 0000000..18686f2 --- /dev/null +++ b/tests/deepeval_tests/api_tool_report_generator.py @@ -0,0 +1,257 @@ +""" +Render the API Tool Calling test results JSON as a Markdown report. + +Reads ``api_tool_test_results.json`` (written by the +``_save_api_tool_results`` autouse fixture in +``tests/deepeval_tests/api_tool_tests.py``) and writes +``api_tool_test_report.md``. The workflow uploads the markdown as an artifact +and posts it as a PR comment. + +This is the API-tool counterpart of ``report_generator.py``; the two are +deliberately independent so changes to the RAG metrics report don't risk +breaking the API-tool report or vice versa. +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import Any, Dict, List, Optional, Tuple + +RESULTS_FILE = Path("api_tool_test_results.json") +REPORT_FILE = Path("api_tool_test_report.md") + + +# --------------------------------------------------------------------------- +# Loading +# --------------------------------------------------------------------------- + + +def load_results(path: Path = RESULTS_FILE) -> Dict[str, Any]: + """Load the JSON written by ApiToolResultCollector.save().""" + if not path.exists(): + return {"error": f"Results file not found: {path}"} + try: + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + except json.JSONDecodeError as e: + return {"error": f"Results file is not valid JSON: {e}"} + + +# --------------------------------------------------------------------------- +# Aggregation helpers +# --------------------------------------------------------------------------- + + +def _by_type(scenarios: List[Dict[str, Any]]) -> Dict[str, List[Dict[str, Any]]]: + grouped: Dict[str, List[Dict[str, Any]]] = {} + for s in scenarios: + grouped.setdefault(s.get("type", "unknown"), []).append(s) + return grouped + + +def _pass_rate(scenarios: List[Dict[str, Any]]) -> Tuple[int, int, float]: + total = len(scenarios) + passed = sum(1 for s in scenarios if s.get("passed")) + rate = (passed / total * 100.0) if total else 0.0 + return passed, total, rate + + +def _status_emoji(scenario: Dict[str, Any]) -> str: + if scenario.get("error"): + return "💥" + if scenario.get("passed"): + return "✅" + return "❌" + + +def _fmt_tool(tc: Optional[Dict[str, Any]]) -> str: + if not tc: + return "_(none)_" + name = tc.get("name", "?") + params = tc.get("input_parameters") or {} + if not params: + return f"`{name}()`" + param_str = ", ".join(f"{k}={v!r}" for k, v in params.items()) + return f"`{name}({param_str})`" + + +def _score_str(score: Optional[float], threshold: float) -> str: + if score is None: + return f"— / {threshold}" + return f"{score:.2f} / {threshold}" + + +# --------------------------------------------------------------------------- +# Report sections +# --------------------------------------------------------------------------- + + +def render_header(results: Dict[str, Any]) -> str: + total = results.get("total_tests", 0) + passed = results.get("passed_tests", 0) + failed = results.get("failed_tests", 0) + errored = results.get("errored_tests", 0) + rate = (passed / total * 100.0) if total else 0.0 + started = results.get("test_start_time", "") + + return ( + "## API Tool Calling Evaluation Report\n\n" + f"_Issue #447 — DeepEval coverage for the API Tool Calling feature._\n\n" + f"**Started:** `{started}`\n\n" + "| Metric | Value |\n" + "|---|---|\n" + f"| Total scenarios | {total} |\n" + f"| Passed | {passed} |\n" + f"| Failed | {failed} |\n" + f"| Errored | {errored} |\n" + f"| Pass rate | **{rate:.1f}%** |\n\n" + ) + + +def render_by_type(results: Dict[str, Any]) -> str: + scenarios = results.get("scenarios", []) + grouped = _by_type(scenarios) + out = "### Results by scenario type\n\n" + out += "| Type | Metric | Passed | Total | Pass rate |\n" + out += "|---|---|---|---|---|\n" + type_metric = { + "strict": "ToolCorrectnessMetric (deterministic, threshold=1.0)", + "loose": "ArgumentCorrectnessMetric (LLM judge, threshold=0.7)", + "multi_intent": "ArgumentCorrectnessMetric or routing-only", + } + type_label = { + "strict": "Strict single-intent (S1, S2a, S2b, S3)", + "loose": "Loose single-intent (S4)", + "multi_intent": "Multi-intent (MI-1..MI-8)", + } + for stype in ("strict", "loose", "multi_intent"): + items = grouped.get(stype, []) + if not items: + continue + p, t, r = _pass_rate(items) + out += ( + f"| {type_label.get(stype, stype)} | " + f"{type_metric.get(stype, '—')} | " + f"{p} | {t} | {r:.1f}% |\n" + ) + return out + "\n" + + +def render_scenario_table(results: Dict[str, Any]) -> str: + scenarios = results.get("scenarios", []) + if not scenarios: + return "### Detailed results\n\n_No scenarios recorded._\n\n" + + out = "### Detailed results\n\n" + out += "| Status | Scenario | Metric | Score | Expected → Actual |\n" + out += "|---|---|---|---|---|\n" + for s in scenarios: + status = _status_emoji(s) + sid = s.get("id", "?") + metric = s.get("metric", "—") + score_cell = _score_str(s.get("score"), s.get("threshold", 0.0)) + expected = _fmt_tool(s.get("expected_tool")) + actual = _fmt_tool(s.get("actual_tool")) + # For multi-intent / loose, expected_tool is None — show "→ {actual}" only + if s.get("expected_tool") is None: + arrow = actual + else: + arrow = f"{expected} → {actual}" + out += f"| {status} | `{sid}` | {metric} | {score_cell} | {arrow} |\n" + return out + "\n" + + +def render_failures(results: Dict[str, Any]) -> str: + failures = [ + s + for s in results.get("scenarios", []) + if not s.get("passed") and not s.get("error") + ] + if not failures: + return "" + out = "### Failed scenarios\n\n" + for s in failures: + sid = s.get("id", "?") + reason = s.get("reason") or "_(no reason captured)_" + preview = (s.get("final_response_preview") or "").replace("\n", " ") + if len(preview) > 200: + preview = preview[:200] + "…" + out += f"#### ❌ `{sid}` ({s.get('metric', '—')})\n\n" + out += f"- **Score:** {_score_str(s.get('score'), s.get('threshold', 0.0))}\n" + if s.get("expected_tool"): + out += f"- **Expected:** {_fmt_tool(s['expected_tool'])}\n" + out += f"- **Actual:** {_fmt_tool(s.get('actual_tool'))}\n" + out += f"- **Reason:** {reason}\n" + if preview: + out += f"- **Response preview:** `{preview}`\n" + out += "\n" + return out + + +def render_errors(results: Dict[str, Any]) -> str: + errored = [s for s in results.get("scenarios", []) if s.get("error")] + if not errored: + return "" + out = "### Errored scenarios (test harness errors, not assertion failures)\n\n" + for s in errored: + sid = s.get("id", "?") + err = s.get("error") or "_(no error captured)_" + out += f"- 💥 `{sid}` — {err}\n" + return out + "\n" + + +def render_methodology() -> str: + return ( + "### Methodology\n\n" + "- **Strict scenarios** (issue specifies the expected endpoint and " + "params) are scored with `ToolCorrectnessMetric`, " + "`evaluation_params=[ToolCallParams.INPUT_PARAMETERS]`, threshold 1.0. " + "Tool name must match exactly and every expected parameter must be " + "present with the expected value (extra parameters allowed).\n" + "- **Loose scenarios** (issue describes the flow but not the expected " + "resolution) are scored with `ArgumentCorrectnessMetric` (LLM-as-judge, " + "threshold 0.7).\n" + "- **Multi-intent scenarios** use `ArgumentCorrectnessMetric` when the " + "agent resolves a tool call; if the agent asks a clarifying question " + "instead, only routing-to-ATC is verified (rules out silent fall-through " + "to RAG/OOD).\n" + "- API tool endpoints are seeded into the testcontainers-backed Qdrant " + "from `tests/api_tool_eval/test-endpoints.json` via the " + "`api_tool_endpoints_indexed` fixture in `conftest.py`.\n\n" + ) + + +def render_report(results: Dict[str, Any]) -> str: + if results.get("error"): + return ( + f"## API Tool Calling Evaluation Report\n\n**ERROR:** {results['error']}\n" + ) + return ( + render_header(results) + + render_by_type(results) + + render_scenario_table(results) + + render_failures(results) + + render_errors(results) + + render_methodology() + ) + + +def main() -> int: + results = load_results() + markdown = render_report(results) + REPORT_FILE.write_text(markdown, encoding="utf-8") + print(f"Wrote {REPORT_FILE} ({len(markdown)} chars)") + if results.get("error"): + print(f" WARNING: {results['error']}", file=sys.stderr) + else: + print( + f" {results.get('passed_tests', 0)}/{results.get('total_tests', 0)} " + "scenarios passed" + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/deepeval_tests/api_tool_tests.py b/tests/deepeval_tests/api_tool_tests.py new file mode 100644 index 0000000..1edaae0 --- /dev/null +++ b/tests/deepeval_tests/api_tool_tests.py @@ -0,0 +1,779 @@ +""" +DeepEval tests for the API Tool Calling feature (issue #447). + +Covers the single-intent (Scenarios 1-4) and multi-intent (MI-1..MI-8) scenarios +listed in the issue against the running orchestration service. Each scenario +walks the ``/orchestrate`` endpoint turn-by-turn via the testcontainers-backed +``orchestration_client`` fixture, extracts the final agentic-loop tool call, +and scores it with a DeepEval agentic metric: + +* **Strict single-intent (S1, S2a, S2b, S3)** — the issue specifies an + "Expected endpoint" + URL/params per scenario. Scored with + ``ToolCorrectnessMetric`` (deterministic name + input-parameter comparison, + ``threshold=1.0``). + +* **Loose single-intent (S4)** — the issue documents the 5-turn flow but does + not specify an expected resolution. Scored with + ``ArgumentCorrectnessMetric`` (LLM-as-judge over the conversation input and + the resolved tool call). + +* **Multi-intent (MI-1..MI-8)** — issue lists only queries (EN+ET). If the + system resolves a tool call, scored with ``ArgumentCorrectnessMetric``; if + it asks a clarifying question instead, the test only verifies routing to + ATC (i.e. non-empty reply, not silent RAG/OOD). + +Endpoints are matched by ``name`` — the UUIDs in the issue do happen to match +those in ``tests/api_tool_eval/test-endpoints.json``, but name matching is the +stable contract. + +Depends on: +* ``orchestration_client`` — provides the testcontainers-mapped base URL. +* ``api_tool_endpoints_indexed`` — seeds the API tool fixture into Qdrant's + ``api_tool_collection`` so the agentic loop can find the endpoints. +""" + +import datetime +import json +import time +import uuid +from pathlib import Path +from typing import Any, Dict, List, Optional + +import pytest +import requests +from deepeval.metrics import ArgumentCorrectnessMetric, ToolCorrectnessMetric +from deepeval.test_case import LLMTestCase, ToolCall, ToolCallParams + +REQUEST_TIMEOUT = 60 +ENVIRONMENT = "development" +AUTHOR_ID = "api-tool-deepeval" + +# Where the result-collector writes the per-scenario record consumed by +# tests/deepeval_tests/api_tool_report_generator.py to render the PR +# comment / artifact markdown. +RESULTS_FILE = Path("api_tool_test_results.json") + +# Strict scenarios assert the exact expected tool was called with the exact +# expected params (extras allowed — see ToolCorrectnessMetric docs on +# should_exact_match). Threshold 1.0 because the deterministic comparison +# scores fractionally over expected_tools, and we want every expected param +# present and correct. +STRICT_TOOL_THRESHOLD = 1.0 + +# Loose scenarios are graded by an LLM judge — 0.7 matches the threshold used +# for the RAG metrics in standard_tests.py. +JUDGE_THRESHOLD = 0.7 + + +# --------------------------------------------------------------------------- +# HTTP helpers (mirror tests/api_tool_eval/integration_test_*.py) +# --------------------------------------------------------------------------- + + +def _make_chat_id(label: str) -> str: + return f"deepeval-api-tool-{label}-{uuid.uuid4().hex[:8]}" + + +def _send_turn( + base_url: str, + chat_id: str, + message: str, + history: List[Dict[str, str]], +) -> Dict[str, Any]: + payload: Dict[str, Any] = { + "chatId": chat_id, + "message": message, + "authorId": AUTHOR_ID, + "conversationHistory": history, + "url": "deepeval-test", + "environment": ENVIRONMENT, + } + resp = requests.post( + f"{base_url}/orchestrate", + json=payload, + timeout=REQUEST_TIMEOUT, + ) + resp.raise_for_status() + return resp.json() + + +def _append_history( + history: List[Dict[str, str]], + user_message: str, + bot_response: str, +) -> List[Dict[str, str]]: + ts = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + return history + [ + {"authorRole": "user", "message": user_message, "timestamp": ts}, + {"authorRole": "bot", "message": bot_response, "timestamp": ts}, + ] + + +def _parse_completed(content: str) -> Optional[Dict[str, Any]]: + """Return the parsed JSON if content is a completed agentic-loop payload, + else None. A completed payload carries both ``endpoint`` and + ``collected_params`` keys.""" + try: + data = json.loads(content) + except (json.JSONDecodeError, TypeError): + return None + if "collected_params" in data and "endpoint" in data: + return data + return None + + +def _to_tool_call(content: str) -> Optional[ToolCall]: + """Build a DeepEval ToolCall from a completed agentic-loop payload, or + None if the response wasn't a completed JSON.""" + data = _parse_completed(content) + if data is None: + return None + ep = data.get("endpoint", {}) + name = ep.get("name", "") if isinstance(ep, dict) else str(ep) + params = data.get("collected_params", {}) or {} + return ToolCall(name=name, input_parameters=params) + + +def _conversation_text(turns: List[Dict[str, Any]]) -> str: + """Flatten the user turns into a single string for LLMTestCase.input. + + DeepEval's agentic single-turn metrics take a single ``input`` string; + this approximation gives the LLM judge the full conversational context. + """ + return "\n".join(f"USER: {t['user']}" for t in turns) + + +def _walk_turns(base_url: str, label: str, turns: List[Dict[str, Any]]) -> str: + """POST each turn in sequence, maintaining a stable chatId + history. + Returns the final bot response content.""" + chat_id = _make_chat_id(label) + history: List[Dict[str, str]] = [] + final_content = "" + for turn in turns: + resp = _send_turn(base_url, chat_id, turn["user"], history) + final_content = resp.get("content", "") + history = _append_history(history, turn["user"], final_content) + return final_content + + +def _tool_call_to_dict(tc: Optional[ToolCall]) -> Optional[Dict[str, Any]]: + """Serialize a ToolCall for the results JSON (None-tolerant).""" + if tc is None: + return None + return {"name": tc.name, "input_parameters": dict(tc.input_parameters or {})} + + +# --------------------------------------------------------------------------- +# Result collector — every test pushes one record, autouse fixture flushes to +# disk at session end. tests/deepeval_tests/api_tool_report_generator.py +# reads the JSON and renders the markdown report consumed by the workflow. +# --------------------------------------------------------------------------- + + +class ApiToolResultCollector: + """Accumulates per-scenario results from the API tool tests.""" + + def __init__(self) -> None: + self.results: Dict[str, Any] = { + "total_tests": 0, + "passed_tests": 0, + "failed_tests": 0, + "errored_tests": 0, + "test_start_time": datetime.datetime.now().isoformat(), + "scenarios": [], + } + + def add( + self, + scenario_id: str, + scenario_type: str, + metric_name: str, + threshold: float, + score: Optional[float], + passed: bool, + reason: str = "", + error: str = "", + expected_tool: Optional[Dict[str, Any]] = None, + actual_tool: Optional[Dict[str, Any]] = None, + final_response_preview: str = "", + extra: Optional[Dict[str, Any]] = None, + ) -> None: + self.results["total_tests"] += 1 + if error: + self.results["errored_tests"] += 1 + elif passed: + self.results["passed_tests"] += 1 + else: + self.results["failed_tests"] += 1 + self.results["scenarios"].append( + { + "id": scenario_id, + "type": scenario_type, + "metric": metric_name, + "threshold": threshold, + "score": score, + "passed": passed, + "reason": reason, + "error": error, + "expected_tool": expected_tool, + "actual_tool": actual_tool, + "final_response_preview": final_response_preview, + "extra": extra or {}, + } + ) + + def save(self, path: Path = RESULTS_FILE) -> None: + with open(path, "w", encoding="utf-8") as f: + json.dump(self.results, f, indent=2, default=str, ensure_ascii=False) + print( + f"Saved API tool results to {path}: " + f"{self.results['passed_tests']}/{self.results['total_tests']} passed, " + f"{self.results['failed_tests']} failed, " + f"{self.results['errored_tests']} errored" + ) + + +_collector = ApiToolResultCollector() + + +@pytest.fixture(scope="session", autouse=True) +def _save_api_tool_results(): + """Flush collected results to RESULTS_FILE at end of session, even on + failure — mirrors save_results_fixture in standard_tests.py.""" + yield + _collector.save() + + +# --------------------------------------------------------------------------- +# Strict single-intent scenarios — issue specifies expected endpoint + params +# (S1, S2a, S2b, S3). Scored with ToolCorrectnessMetric. +# --------------------------------------------------------------------------- + + +STRICT_SINGLE_INTENT_SCENARIOS: List[Dict[str, Any]] = [ + # Scenario 1 — Normal Workflow (citizen initiative details) + { + "id": "S1-citizen-initiative-EN", + "label": "s1-en", + "turns": [ + {"user": "Can I see the details of a citizen initiative?"}, + {"user": "1790"}, + ], + "expected_tool": ToolCall( + name="get_initiative_details", + input_parameters={"id": "1790"}, + ), + }, + { + "id": "S1-citizen-initiative-ET", + "label": "s1-et", + "turns": [ + {"user": "Kas ma saan kodanikualgatuse üksikasju vaadata?"}, + {"user": "1790"}, + ], + "expected_tool": ToolCall( + name="get_initiative_details", + input_parameters={"id": "1790"}, + ), + }, + # Scenario 2a — Public Holidays (date range correction) + { + "id": "S2a-public-holidays-date-correction-EN", + "label": "s2a-en", + "turns": [ + {"user": "What are the public holidays in Estonia?"}, + {"user": "From 2026-01-01"}, + { + "user": ( + "My mistake — the correct period is April 1, 2026 " + "through December 31, 2026." + ) + }, + ], + "expected_tool": ToolCall( + name="get_public_holidays", + input_parameters={ + "countryIsoCode": "EE", + "validFrom": "2026-04-01", + "validTo": "2026-12-31", + }, + ), + }, + { + "id": "S2a-public-holidays-date-correction-ET", + "label": "s2a-et", + "turns": [ + {"user": "Millised on riigipühad Eestis?"}, + {"user": "Alates 1. jaanuarist 2026"}, + { + "user": ( + "Minu viga, õige periood on 01.04.2026 kuni 31.12.2026. " + "Tegelikult tahan 2026-04-01 kuni 2026-12-31." + ) + }, + ], + "expected_tool": ToolCall( + name="get_public_holidays", + input_parameters={ + "countryIsoCode": "EE", + "validFrom": "2026-04-01", + "validTo": "2026-12-31", + }, + ), + }, + # Scenario 2b — Parliament Votings (date range correction) + { + "id": "S2b-parliament-votings-date-correction-EN", + "label": "s2b-en", + "turns": [ + {"user": "What votes took place in the Estonian parliament?"}, + {"user": "2026-04-05"}, + { + "user": ( + "My mistake — the correct period is April 6, 2026 " + "through April 7, 2026." + ) + }, + ], + "expected_tool": ToolCall( + name="get_parliament_votings", + input_parameters={ + "startDate": "2026-04-06", + "endDate": "2026-04-07", + }, + ), + }, + { + "id": "S2b-parliament-votings-date-correction-DE", + "label": "s2b-de", + "turns": [ + {"user": ("Welche Abstimmungen fanden im estnischen Parlament statt?")}, + {"user": "2026-04-05"}, + { + "user": ( + "Mein Fehler — der richtige Zeitraum ist vom " + "6. April 2026 bis zum 7. April 2026." + ) + }, + ], + "expected_tool": ToolCall( + name="get_parliament_votings", + input_parameters={ + "startDate": "2026-04-06", + "endDate": "2026-04-07", + }, + ), + }, + # Scenario 3 — Intent Switch (electricity prices -> address search) + { + "id": "S3-intent-switch-EN", + "label": "s3-en", + "turns": [ + {"user": "Show last week's electricity prices in Estonia."}, + { + "user": ( + "Wait — could you check the following location instead: " + "Viru tn 4, Tallinn?" + ) + }, + ], + "expected_tool": ToolCall( + name="search_address", + input_parameters={"address": "Viru tn 4, Tallinn"}, + ), + }, + { + "id": "S3-intent-switch-ET", + "label": "s3-et", + "turns": [ + {"user": "Näita eelmise nädala elektrienergia hindu Eestis."}, + { + "user": ( + "Oota, kas saaksid hoopis järgmist asukohta kontrollida: " + "Viru tn 4, Tallinn?" + ) + }, + ], + "expected_tool": ToolCall( + name="search_address", + input_parameters={"address": "Viru tn 4, Tallinn"}, + ), + }, +] + + +@pytest.mark.parametrize( + "scenario", + STRICT_SINGLE_INTENT_SCENARIOS, + ids=[s["id"] for s in STRICT_SINGLE_INTENT_SCENARIOS], +) +def test_api_tool_strict_single_intent( + scenario: Dict[str, Any], + orchestration_client: Any, + api_tool_endpoints_indexed: None, +) -> None: + """Scenarios 1, 2a, 2b, 3 — issue specifies the expected tool call. + + Scored deterministically with ``ToolCorrectnessMetric``: + * the resolved tool's name must equal the expected name, and + * every expected input parameter must be present and equal in the + ``collected_params`` (extras allowed). + """ + del api_tool_endpoints_indexed # consumed for its setup side effect only + + expected_tool: ToolCall = scenario["expected_tool"] + final_content = "" + actual_tool: Optional[ToolCall] = None + score: Optional[float] = None + reason = "" + passed = False + error = "" + + try: + final_content = _walk_turns( + orchestration_client.base_url, scenario["label"], scenario["turns"] + ) + actual_tool = _to_tool_call(final_content) + test_case = LLMTestCase( + input=_conversation_text(scenario["turns"]), + actual_output=final_content, + tools_called=[actual_tool] if actual_tool is not None else [], + expected_tools=[expected_tool], + ) + metric = ToolCorrectnessMetric( + threshold=STRICT_TOOL_THRESHOLD, + evaluation_params=[ToolCallParams.INPUT_PARAMETERS], + ) + metric.measure(test_case) + score = metric.score + reason = metric.reason or "" + passed = score is not None and score >= STRICT_TOOL_THRESHOLD + assert passed, ( + f"[{scenario['id']}] tool correctness {score} < " + f"{STRICT_TOOL_THRESHOLD}: {reason}\n" + f"Expected tool: {expected_tool.name}({expected_tool.input_parameters})\n" + f"Final response: {final_content[:300]}" + ) + except AssertionError: + raise + except Exception as e: + error = f"{type(e).__name__}: {e}" + raise + finally: + _collector.add( + scenario_id=scenario["id"], + scenario_type="strict", + metric_name="ToolCorrectnessMetric", + threshold=STRICT_TOOL_THRESHOLD, + score=score, + passed=passed, + reason=reason, + error=error, + expected_tool=_tool_call_to_dict(expected_tool), + actual_tool=_tool_call_to_dict(actual_tool), + final_response_preview=final_content[:300], + ) + + +# --------------------------------------------------------------------------- +# Loose single-intent scenarios — issue documents the flow but does not +# specify an "Expected endpoint" (S4, both languages). Scored with +# ArgumentCorrectnessMetric (LLM judges arg correctness vs. the +# conversation input). +# --------------------------------------------------------------------------- + + +LOOSE_SINGLE_INTENT_SCENARIOS: List[Dict[str, Any]] = [ + { + "id": "S4-parliament-attendance-multi-turn-EN", + "label": "s4-en", + "turns": [ + { + "user": ( + "Can you show me the parliament attendance of former " + "Finance Minister Martin Helme?" + ) + }, + {"user": "Can you just check with what you have?"}, + {"user": "2026-04-01"}, + {"user": "Yes"}, + {"user": "2026-04-20"}, + ], + }, + { + "id": "S4-parliament-attendance-multi-turn-ET", + "label": "s4-et", + "turns": [ + { + "user": ( + "Kas saaksite mulle näidata endise rahandusministri " + "Martin Helme parlamendi kohaloleku andmeid?" + ) + }, + {"user": "Kas saaksite lihtsalt oma andmetest järele vaadata?"}, + {"user": "2026-04-01"}, + {"user": "Jah"}, + {"user": "2026-04-20"}, + ], + }, +] + + +@pytest.mark.parametrize( + "scenario", + LOOSE_SINGLE_INTENT_SCENARIOS, + ids=[s["id"] for s in LOOSE_SINGLE_INTENT_SCENARIOS], +) +def test_api_tool_loose_single_intent( + scenario: Dict[str, Any], + orchestration_client: Any, + api_tool_endpoints_indexed: None, +) -> None: + """Scenario 4 (EN/ET) — issue gives no expected resolution. + + Asserts: + 1. The final turn produced a completed JSON tool call (i.e. the agentic + loop actually resolved, didn't fall through to RAG). + 2. The LLM judge ``ArgumentCorrectnessMetric`` is satisfied that the + chosen tool's arguments fit the conversation input. + """ + del api_tool_endpoints_indexed # consumed for its setup side effect only + + final_content = "" + actual_tool: Optional[ToolCall] = None + score: Optional[float] = None + reason = "" + passed = False + error = "" + + try: + final_content = _walk_turns( + orchestration_client.base_url, scenario["label"], scenario["turns"] + ) + actual_tool = _to_tool_call(final_content) + + assert actual_tool is not None, ( + f"[{scenario['id']}] expected a completed JSON tool call on the " + f"final turn but got: {final_content[:300]}" + ) + + test_case = LLMTestCase( + input=_conversation_text(scenario["turns"]), + actual_output=final_content, + tools_called=[actual_tool], + ) + metric = ArgumentCorrectnessMetric(threshold=JUDGE_THRESHOLD) + metric.measure(test_case) + score = metric.score + reason = metric.reason or "" + passed = score is not None and score >= JUDGE_THRESHOLD + assert passed, ( + f"[{scenario['id']}] argument correctness {score} < " + f"{JUDGE_THRESHOLD}: {reason}\n" + f"Resolved tool: {actual_tool.name}({actual_tool.input_parameters})" + ) + except AssertionError: + raise + except Exception as e: + error = f"{type(e).__name__}: {e}" + raise + finally: + _collector.add( + scenario_id=scenario["id"], + scenario_type="loose", + metric_name="ArgumentCorrectnessMetric", + threshold=JUDGE_THRESHOLD, + score=score, + passed=passed, + reason=reason, + error=error, + actual_tool=_tool_call_to_dict(actual_tool), + final_response_preview=final_content[:300], + ) + + +# --------------------------------------------------------------------------- +# Multi-Intent scenarios (issue #447, MI-1..MI-8) +# +# Issue lists 8 queries (EN + ET) with no expected resolution. Under Phase 1 +# the orchestrator decomposes the query and falls back to a single endpoint +# (see tests/api_tool_eval/integration_test_multi_intent.py docstring). We: +# +# * If the system resolves a tool call → score with ArgumentCorrectnessMetric +# (LLM judges whether the chosen args fit the multi-intent query). +# * If the system asks a clarifying question → accept it (still ATC-routed), +# only assert the reply is non-empty (rules out a silent RAG/OOD fallthrough). +# --------------------------------------------------------------------------- + + +MULTI_INTENT_SCENARIOS: List[Dict[str, Any]] = [ + { + "id": "MI-1-address-and-vehicle-tax", + "label": "mi1", + "query_en": ( + "Can you find an address for me and also calculate my vehicle " + "tax? (Address: Viru tn 4, Tallinn / Plate: 123ABC / Year: 2026)" + ), + "query_et": ( + "Kas saaksite mulle aadressi leida ja arvutada ka mu sõiduki " + "maksu? (Aadress: Viru tn 4, Tallinn / Registreerimismärk: " + "123ABC / Aasta: 2026)" + ), + }, + { + "id": "MI-2-address-and-initiative-details", + "label": "mi2", + "query_en": ( + "I need to find an address and also check details of an " + "initiative. (Address: Viru tn 4, Tallinn / Initiative ID: 1790)" + ), + "query_et": ( + "Mul on vaja leida aadress ja vaadata ka kodanikualgatuse " + "üksikasju. (Aadress: Viru tn 4, Tallinn / Algatuse ID: 1790)" + ), + }, + { + "id": "MI-3-electricity-and-public-holidays", + "label": "mi3", + "query_en": ( + "Show me electricity prices in Estonia and list Estonia's public holidays." + ), + "query_et": ("Näita mulle Eesti elektrihindu ja too välja Eesti riigipühad."), + }, + { + "id": "MI-4-parliament-votings-and-initiatives", + "label": "mi4", + "query_en": ( + "Show me the parliament voting results and also list the " + "citizen initiatives." + ), + "query_et": ( + "Näita mulle parlamendi hääletustulemusi ja too välja ka kodanikualgatused." + ), + }, + { + "id": "MI-5-address-and-parliament-participation", + "label": "mi5", + "query_en": ( + "Could you find me an address and also show the attendance " + "statistics of Riigikogu members?" + ), + "query_et": ( + "Kas saaksite mulle leida aadressi ja näidata ka Riigikogu " + "liikmete osalusstatistikat?" + ), + }, + { + "id": "MI-6-public-holidays-and-vehicle-tax", + "label": "mi6", + "query_en": ( + "What are the public holidays in Estonia and can you also " + "calculate my vehicle tax?" + ), + "query_et": ( + "Millised on riigipühad Eestis ja kas saate arvutada ka mu sõiduki maksu?" + ), + }, + { + "id": "MI-7-initiatives-and-parliament-votings", + "label": "mi7", + "query_en": ( + "Show me all citizen initiatives and also display the parliament " + "voting results." + ), + "query_et": ( + "Kuva mulle kõik kodanikualgatused ja näita ka riigikogu hääletustulemusi." + ), + }, + { + "id": "MI-8-vehicle-tax-and-electricity", + "label": "mi8", + "query_en": ( + "Calculate my vehicle tax and also show me the electricity " + "market price in Estonia." + ), + "query_et": ( + "Arvuta mu sõiduki maks ja näita mulle ka Eesti elektri turuhinda." + ), + }, +] + + +@pytest.mark.parametrize( + "scenario", + MULTI_INTENT_SCENARIOS, + ids=[s["id"] for s in MULTI_INTENT_SCENARIOS], +) +@pytest.mark.parametrize("lang", ["en", "et"]) +def test_api_tool_multi_intent( + scenario: Dict[str, Any], + lang: str, + orchestration_client: Any, + api_tool_endpoints_indexed: None, +) -> None: + del api_tool_endpoints_indexed # consumed for its setup side effect only + + query = scenario[f"query_{lang}"] + content = "" + actual_tool: Optional[ToolCall] = None + score: Optional[float] = None + reason = "" + passed = False + error = "" + outcome = "unknown" # "tool_call" | "clarifying_question" + + try: + base_url = orchestration_client.base_url + chat_id = _make_chat_id(f"{scenario['label']}-{lang}") + resp = _send_turn(base_url, chat_id, query, []) + content = resp.get("content", "") + actual_tool = _to_tool_call(content) + + if actual_tool is not None: + outcome = "tool_call" + test_case = LLMTestCase( + input=query, + actual_output=content, + tools_called=[actual_tool], + ) + metric = ArgumentCorrectnessMetric(threshold=JUDGE_THRESHOLD) + metric.measure(test_case) + score = metric.score + reason = metric.reason or "" + passed = score is not None and score >= JUDGE_THRESHOLD + assert passed, ( + f"[{scenario['id']} {lang}] argument correctness {score} " + f"< {JUDGE_THRESHOLD}: {reason}\n" + f"Resolved tool: {actual_tool.name}({actual_tool.input_parameters})" + ) + else: + outcome = "clarifying_question" + # No tool call yet — must at least be a non-empty clarifying reply + # (rules out silent failure / RAG fallthrough returning no content). + passed = bool(content.strip()) + assert passed, ( + f"[{scenario['id']} {lang}] empty response — multi-intent query " + f"failed routing entirely. Got: {content!r}" + ) + reason = "Resolved to a clarifying question (no tool call yet)" + except AssertionError: + raise + except Exception as e: + error = f"{type(e).__name__}: {e}" + raise + finally: + _collector.add( + scenario_id=f"{scenario['id']}-{lang}", + scenario_type="multi_intent", + metric_name=( + "ArgumentCorrectnessMetric" if outcome == "tool_call" else "RoutingOnly" + ), + threshold=JUDGE_THRESHOLD if outcome == "tool_call" else 0.0, + score=score, + passed=passed, + reason=reason, + error=error, + actual_tool=_tool_call_to_dict(actual_tool), + final_response_preview=content[:300], + extra={"language": lang, "outcome": outcome, "query": query}, + ) diff --git a/tests/deepeval_tests/conftest.py b/tests/deepeval_tests/conftest.py index 660778c..7bb40f2 100644 --- a/tests/deepeval_tests/conftest.py +++ b/tests/deepeval_tests/conftest.py @@ -371,7 +371,7 @@ def _write_test_secrets(self, client: hvac.Client) -> None: "endpoint": azure_endpoint, "api_key": azure_api_key, "deployment_name": azure_deployment or "gpt-4o-mini", - "environment": "testing", + "environment": "development", "model": "gpt-4o-mini", "model_type": "chat", "api_version": "2024-02-15-preview", @@ -384,11 +384,11 @@ def _write_test_secrets(self, client: hvac.Client) -> None: client.secrets.kv.v2.create_or_update_secret( mount_point="secret", - path="llm/connections/azure_openai/evalconnection-1", + path="llm/connections/azure_openai/development/evalconnection-1", secret=llm_secret, ) logger.info( - "LLM connection secret written to llm/connections/azure_openai/evalconnection-1" + "LLM connection secret written to llm/connections/azure_openai/development/evalconnection-1" ) # ============================================================ @@ -401,7 +401,7 @@ def _write_test_secrets(self, client: hvac.Client) -> None: "endpoint": azure_endpoint, "api_key": azure_api_key, "deployment_name": azure_embedding_deployment, # This is the embedding deployment - "environment": "testing", + "environment": "development", "model": "text-embedding-3-large", "model_type": "embedding", "api_version": "2024-02-15-preview", @@ -413,17 +413,17 @@ def _write_test_secrets(self, client: hvac.Client) -> None: logger.info(f" → model: {embedding_secret['model']}") logger.info(f" → connection_id: {embedding_secret['connection_id']}") logger.info( - " → Vault path: embeddings/connections/azure_openai/evalconnection-1" + " → Vault path: embeddings/connections/azure_openai/development/evalconnection-1" ) # Write to embeddings path with connection_id in the path client.secrets.kv.v2.create_or_update_secret( mount_point="secret", - path="embeddings/connections/azure_openai/evalconnection-1", + path="embeddings/connections/azure_openai/development/evalconnection-1", secret=embedding_secret, ) logger.info( - "Embedding secret written to embeddings/connections/azure_openai/evalconnection-1" + "Embedding secret written to embeddings/connections/azure_openai/development/evalconnection-1" ) # ============================================================ @@ -796,3 +796,106 @@ def __init__(self, base_url: str): self.base_url = base_url return OrchestrationClient(rag_stack.get_orchestration_service_url()) + + +@pytest.fixture(scope="session") +def api_tool_endpoints_indexed(rag_stack: RAGStackTestContainers): + """ + Session-scoped fixture that seeds the API tool endpoints from + ``tests/api_tool_eval/test-endpoints.json`` into Qdrant's + ``api_tool_collection`` so the agentic loop can find them. + + Required by tests in ``tests/deepeval_tests/api_tool_tests.py`` — without + it, semantic search over API tools returns nothing and the orchestrator + falls back to RAG. + + The indexer module hard-codes container-internal URLs in its constants + (``http://llm-orchestration-service:8100``, ``qdrant:6333``). Two of the + three are used in code paths that re-read the constant at call time + (monkey-patching the class attribute works), but ``ApiToolQdrantManager`` + is instantiated with no arguments inside ``index_endpoint`` and binds the + host/port at function-definition time as defaults. To override those, the + fixture also replaces the ``ApiToolQdrantManager`` name imported into + ``main_indexer`` with a factory that injects the testcontainers-mapped + host/port. + """ + import asyncio + import json as _json + from pathlib import Path as _Path + from unittest.mock import patch + from urllib.parse import urlparse + + from api_tool_indexer import main_indexer + from api_tool_indexer.constants import ApiToolIndexerConstants + from api_tool_indexer.main_indexer import index_endpoint + from api_tool_indexer.models import EndpointData + from api_tool_indexer.qdrant_manager import ApiToolQdrantManager + + orch_url = rag_stack.get_orchestration_service_url() + qdrant_url = rag_stack.get_qdrant_url() + parsed = urlparse(qdrant_url) + qdrant_host = parsed.hostname or "localhost" + qdrant_port = parsed.port or 6333 + + def _qdrant_factory(*args: Any, **kwargs: Any) -> ApiToolQdrantManager: + kwargs.setdefault("host", qdrant_host) + kwargs.setdefault("port", qdrant_port) + return ApiToolQdrantManager(*args, **kwargs) + + endpoints_file = ( + _Path(__file__).parent.parent / "api_tool_eval" / "test-endpoints.json" + ) + if not endpoints_file.exists(): + pytest.skip( + f"API tool endpoint fixture not found: {endpoints_file} — " + "cannot seed api_tool_collection" + ) + + with open(endpoints_file, encoding="utf-8") as f: + endpoints = _json.load(f) + + async def _seed_all() -> list: + results = [] + for ep in endpoints: + endpoint_data = EndpointData( + endpoint_id=ep["endpointId"], + name=ep["name"], + description=ep["description"], + url=ep["url"], + method=ep["method"], + params=ep.get("params", []), + ) + res = await index_endpoint(endpoint_data) + logger.info( + f"Seeded endpoint '{ep['name']}': success={res.success} ({res.message})" + ) + results.append((ep["name"], res)) + return results + + # Connection ID "evalconnection-1" matches the one used by standard_tests.py + # and the EVAL_MODE setup; vault is seeded for it. The default + # "gpt-4o-mini" used by the indexer in production has no test fixture. + with ( + patch.object(ApiToolIndexerConstants, "DEFAULT_API_BASE_URL", orch_url), + patch.object( + ApiToolIndexerConstants, + "DEFAULT_CONNECTION_ID", + "evalconnection-1", + ), + patch.object(main_indexer, "ApiToolQdrantManager", _qdrant_factory), + ): + logger.info( + f"Seeding {len(endpoints)} API tool endpoints " + f"(orch={orch_url}, qdrant={qdrant_host}:{qdrant_port})" + ) + results = asyncio.run(_seed_all()) + + failed = [name for name, r in results if not r.success] + if failed: + pytest.skip( + f"API tool seeding failed for {len(failed)}/{len(endpoints)} " + f"endpoints ({failed}) — skipping API tool tests" + ) + + logger.info(f"Indexed all {len(endpoints)} API tool endpoints successfully") + yield From a08ab1d375e2dc214ec326aea041fc3c46867881 Mon Sep 17 00:00:00 2001 From: nuwangeek Date: Thu, 25 Jun 2026 17:19:48 +0530 Subject: [PATCH 2/5] updated deepeval branch --- .github/workflows/deepeval-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deepeval-tests.yml b/.github/workflows/deepeval-tests.yml index 5ed338a..76756ad 100644 --- a/.github/workflows/deepeval-tests.yml +++ b/.github/workflows/deepeval-tests.yml @@ -3,7 +3,7 @@ name: DeepEval RAG System Tests on: pull_request: types: [opened, synchronize, reopened] - branches: ["wip-eval"] + branches: ["deepeval-temp"] paths: - 'src/**' - 'tests/**' From d74b1d22dc4b5a700ca08d00775a1d61babaed60 Mon Sep 17 00:00:00 2001 From: nuwangeek Date: Fri, 26 Jun 2026 06:19:21 +0530 Subject: [PATCH 3/5] reduced dataset --- tests/data/test_dataset.json | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/tests/data/test_dataset.json b/tests/data/test_dataset.json index 431ab1e..1598ebc 100644 --- a/tests/data/test_dataset.json +++ b/tests/data/test_dataset.json @@ -34,29 +34,5 @@ "expected_output": "Eesti sotsiaaluuring aitab hinnata leibkondade ja isikute sissetulekute jaotust, elamistingimusi ning sotsiaalset tõrjutust. Samasugust uuringut korraldatakse kõikides Euroopa Liidu riikides. Teie osalemine aitab kujundada Eesti sotsiaalpoliitikat ja arendada teenuseid. Miks see on oluline: Need andmed aitavad riigijuhtidel, teadlastel ja arvamusliidridel mõista inimeste toimetulekut, vaesust, ilmajäetust, sissetulekut, elamistingimusi ja arstiabi kättesaadavust. Iga uuringusse sattunud inimene esindab umbes sadat Eesti elanikku, seega on teie vastused olulised. Kuidas osaleda: Kui teid valitakse, saate teavituskirja e-postile (rahvastikuregistris märgitud aadressile) või postiga. Saate küsimustikule vastata veebis aadressil https://estat.stat.ee, logides sisse ID-kaardi, mobiil-ID või Smart-ID-ga, või osaleda telefoniintervjuus. Uuring koosneb kahest osast: leibkonnaküsimustik (vastab põhiisik) ja isikuküsimustik (võib vastata iga vähemalt 15-aastane leibkonnaliige). Osalemine on vabatahtlik, kuid Statistikaamet kaitseb kõigi vastajate andmeid riikliku statistika seaduse ja isikuandmete kaitse seaduse alusel. Iga leibkonda küsitletakse neli aastat järjest 12 kuu tagant.", "category": "statistics", "language": "et" - }, - { - "input": "Kas ma saan kodus elektritöid ise teha või vajan spetsialisti?", - "expected_output": "Enamik elektritöid nõuab ohutuse huvides kvalifitseeritud spetsialisti. Valesti tehtud elektritöö võib põhjustada elektrilöögi ning tulekahjuohtu. Siiski saate lihtsamaid töid ise teha, kui teil on vastavad teadmised. Mida VÕITE ise teha: Vahetada lüliteid, pistikupesi, lambipesi ja kaitsmeid (kuid MITTE paigaldada uusi) Parandada ja asendada juhtmelüliteid, lambipesi, pikendusjuhtmeid ja juhtmepistikuid Milleks PEATE palkama spetsialisti: Uute elektripaigaldiste ehitamine Uute pistikupesade ja lülitite paigaldamine Kohtkindlate kodumasinate ühendamine ja lahti ühendamine Kaitsekontaktita (maandamata) pistikupesade vahetamine kaitsekontaktiga (maandatud) pistikupesade vastu Elektritöö ettevõtjad peavad olema esitanud majandustegevuse registrisse majandustegevuseteatise ning neil peab olema tööde eest vastutav kompetentne elektritöö juht.", - "category": "ttja", - "language": "et" - }, - { - "input": "What is an electrical installation audit and when do I need one?", - "expected_output": "An electrical installation audit checks whether your electrical system meets safety requirements and is safe to use. During the audit, the auditor visually assesses the installation's condition, reviews documentation and test/measurement results, and performs additional control measurements if necessary. When you need an audit: Before commissioning: Required before putting a new or renovated building's electrical installation into use Periodic audits: Regular checks at intervals depending on the installation type and age. While not mandatory for residential spaces (private houses, apartments, summer cottages), you should still periodically check them to ensure safety and functionality How to get an audit: Only contractors with appropriate accreditation can perform audits Results and documents are digitally formatted in TTJA's information system at https://jvis.ttja.ee, where they're always accessible to the electrical installation owner For residential electrical system checks, contact a competent electrical professional or auditor who will perform necessary operations and provide feedback on the system's condition and safety.", - "category": "ttja", - "language": "en" - }, - { - "input": "How long is the e-residency digi-ID valid for?", - "expected_output": "The e-residency digi-ID is valid for 5 years", - "category": "digital_identity", - "language": "en" - }, - { - "input": "Предоставляет ли электронное резидентство эстонское гражданство или налоговое резидентство?", - "expected_output": "Нет, электронное резидентство не предоставляет эстонское гражданство или налоговое резидентство.", - "category": "digital_identity", - "language": "ru" } ] \ No newline at end of file From 43e45657c0c95d1f2a6a031af92dcfe080b5cf9c Mon Sep 17 00:00:00 2001 From: nuwangeek Date: Fri, 26 Jun 2026 07:12:01 +0530 Subject: [PATCH 4/5] fixed issue --- .github/workflows/deepeval-tests.yml | 67 +- .../api_tool_report_generator.py | 257 ------ tests/deepeval_tests/api_tool_tests.py | 779 ------------------ tests/deepeval_tests/conftest.py | 103 --- 4 files changed, 6 insertions(+), 1200 deletions(-) delete mode 100644 tests/deepeval_tests/api_tool_report_generator.py delete mode 100644 tests/deepeval_tests/api_tool_tests.py diff --git a/.github/workflows/deepeval-tests.yml b/.github/workflows/deepeval-tests.yml index 76756ad..b30c85e 100644 --- a/.github/workflows/deepeval-tests.yml +++ b/.github/workflows/deepeval-tests.yml @@ -209,13 +209,8 @@ jobs: ENCRYPTION_KEY: ${{ secrets.ENCRYPTION_KEY }} SALT: ${{ secrets.SALT }} run: | - # Run tests sequentially (one at a time) to avoid rate limiting. - # standard_tests.py — RAG quality (DeepEval metrics on /orchestrate-eval) - # api_tool_tests.py — API Tool Calling scenarios (issue #447) - uv run python -m pytest \ - tests/deepeval_tests/standard_tests.py \ - tests/deepeval_tests/api_tool_tests.py \ - -v --tb=short --log-cli-level=INFO -n 0 + # Run tests sequentially (one at a time) to avoid rate limiting + uv run python -m pytest tests/deepeval_tests/standard_tests.py -v --tb=short --log-cli-level=INFO -n 0 - name: Fix permissions on test artifacts if: always() @@ -226,11 +221,7 @@ jobs: - name: Generate evaluation report if: always() run: uv run python tests/deepeval_tests/report_generator.py - - - name: Generate API tool evaluation report - if: always() - run: uv run python tests/deepeval_tests/api_tool_report_generator.py - + - name: Save test artifacts if: always() uses: actions/upload-artifact@v4 @@ -239,10 +230,8 @@ jobs: path: | pytest_captured_results.json test_report.md - api_tool_test_results.json - api_tool_test_report.md retention-days: 30 - + - name: Comment PR with test results if: always() && github.event_name == 'pull_request' uses: actions/github-script@v7 @@ -256,12 +245,12 @@ jobs: repo: context.repo.repo, issue_number: context.issue.number }); - + const existingComment = comments.data.find( comment => comment.user.login === 'github-actions[bot]' && comment.body.includes('RAG System Evaluation Report') ); - + if (existingComment) { await github.rest.issues.updateComment({ owner: context.repo.owner, @@ -286,50 +275,6 @@ jobs: body: `## RAG System Evaluation Report\n\n**Error generating test report**\n\nFailed to read or post test results. Check workflow logs for details.\n\nError: ${error.message}` }); } - - - name: Comment PR with API tool test results - if: always() && github.event_name == 'pull_request' - uses: actions/github-script@v7 - with: - script: | - const fs = require('fs'); - try { - const reportContent = fs.readFileSync('api_tool_test_report.md', 'utf8'); - const comments = await github.rest.issues.listComments({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number - }); - - const existingComment = comments.data.find( - comment => comment.user.login === 'github-actions[bot]' && - comment.body.includes('API Tool Calling Evaluation Report') - ); - - if (existingComment) { - await github.rest.issues.updateComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: existingComment.id, - body: reportContent - }); - } else { - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - body: reportContent - }); - } - } catch (error) { - console.error('Failed to post API tool test results:', error); - await github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: `## API Tool Calling Evaluation Report\n\n**Error generating report**\n\nFailed to read or post API tool test results. Check workflow logs.\n\nError: ${error.message}` - }); - } - name: Check test results and fail if needed if: always() diff --git a/tests/deepeval_tests/api_tool_report_generator.py b/tests/deepeval_tests/api_tool_report_generator.py deleted file mode 100644 index 18686f2..0000000 --- a/tests/deepeval_tests/api_tool_report_generator.py +++ /dev/null @@ -1,257 +0,0 @@ -""" -Render the API Tool Calling test results JSON as a Markdown report. - -Reads ``api_tool_test_results.json`` (written by the -``_save_api_tool_results`` autouse fixture in -``tests/deepeval_tests/api_tool_tests.py``) and writes -``api_tool_test_report.md``. The workflow uploads the markdown as an artifact -and posts it as a PR comment. - -This is the API-tool counterpart of ``report_generator.py``; the two are -deliberately independent so changes to the RAG metrics report don't risk -breaking the API-tool report or vice versa. -""" - -from __future__ import annotations - -import json -import sys -from pathlib import Path -from typing import Any, Dict, List, Optional, Tuple - -RESULTS_FILE = Path("api_tool_test_results.json") -REPORT_FILE = Path("api_tool_test_report.md") - - -# --------------------------------------------------------------------------- -# Loading -# --------------------------------------------------------------------------- - - -def load_results(path: Path = RESULTS_FILE) -> Dict[str, Any]: - """Load the JSON written by ApiToolResultCollector.save().""" - if not path.exists(): - return {"error": f"Results file not found: {path}"} - try: - with open(path, "r", encoding="utf-8") as f: - return json.load(f) - except json.JSONDecodeError as e: - return {"error": f"Results file is not valid JSON: {e}"} - - -# --------------------------------------------------------------------------- -# Aggregation helpers -# --------------------------------------------------------------------------- - - -def _by_type(scenarios: List[Dict[str, Any]]) -> Dict[str, List[Dict[str, Any]]]: - grouped: Dict[str, List[Dict[str, Any]]] = {} - for s in scenarios: - grouped.setdefault(s.get("type", "unknown"), []).append(s) - return grouped - - -def _pass_rate(scenarios: List[Dict[str, Any]]) -> Tuple[int, int, float]: - total = len(scenarios) - passed = sum(1 for s in scenarios if s.get("passed")) - rate = (passed / total * 100.0) if total else 0.0 - return passed, total, rate - - -def _status_emoji(scenario: Dict[str, Any]) -> str: - if scenario.get("error"): - return "💥" - if scenario.get("passed"): - return "✅" - return "❌" - - -def _fmt_tool(tc: Optional[Dict[str, Any]]) -> str: - if not tc: - return "_(none)_" - name = tc.get("name", "?") - params = tc.get("input_parameters") or {} - if not params: - return f"`{name}()`" - param_str = ", ".join(f"{k}={v!r}" for k, v in params.items()) - return f"`{name}({param_str})`" - - -def _score_str(score: Optional[float], threshold: float) -> str: - if score is None: - return f"— / {threshold}" - return f"{score:.2f} / {threshold}" - - -# --------------------------------------------------------------------------- -# Report sections -# --------------------------------------------------------------------------- - - -def render_header(results: Dict[str, Any]) -> str: - total = results.get("total_tests", 0) - passed = results.get("passed_tests", 0) - failed = results.get("failed_tests", 0) - errored = results.get("errored_tests", 0) - rate = (passed / total * 100.0) if total else 0.0 - started = results.get("test_start_time", "") - - return ( - "## API Tool Calling Evaluation Report\n\n" - f"_Issue #447 — DeepEval coverage for the API Tool Calling feature._\n\n" - f"**Started:** `{started}`\n\n" - "| Metric | Value |\n" - "|---|---|\n" - f"| Total scenarios | {total} |\n" - f"| Passed | {passed} |\n" - f"| Failed | {failed} |\n" - f"| Errored | {errored} |\n" - f"| Pass rate | **{rate:.1f}%** |\n\n" - ) - - -def render_by_type(results: Dict[str, Any]) -> str: - scenarios = results.get("scenarios", []) - grouped = _by_type(scenarios) - out = "### Results by scenario type\n\n" - out += "| Type | Metric | Passed | Total | Pass rate |\n" - out += "|---|---|---|---|---|\n" - type_metric = { - "strict": "ToolCorrectnessMetric (deterministic, threshold=1.0)", - "loose": "ArgumentCorrectnessMetric (LLM judge, threshold=0.7)", - "multi_intent": "ArgumentCorrectnessMetric or routing-only", - } - type_label = { - "strict": "Strict single-intent (S1, S2a, S2b, S3)", - "loose": "Loose single-intent (S4)", - "multi_intent": "Multi-intent (MI-1..MI-8)", - } - for stype in ("strict", "loose", "multi_intent"): - items = grouped.get(stype, []) - if not items: - continue - p, t, r = _pass_rate(items) - out += ( - f"| {type_label.get(stype, stype)} | " - f"{type_metric.get(stype, '—')} | " - f"{p} | {t} | {r:.1f}% |\n" - ) - return out + "\n" - - -def render_scenario_table(results: Dict[str, Any]) -> str: - scenarios = results.get("scenarios", []) - if not scenarios: - return "### Detailed results\n\n_No scenarios recorded._\n\n" - - out = "### Detailed results\n\n" - out += "| Status | Scenario | Metric | Score | Expected → Actual |\n" - out += "|---|---|---|---|---|\n" - for s in scenarios: - status = _status_emoji(s) - sid = s.get("id", "?") - metric = s.get("metric", "—") - score_cell = _score_str(s.get("score"), s.get("threshold", 0.0)) - expected = _fmt_tool(s.get("expected_tool")) - actual = _fmt_tool(s.get("actual_tool")) - # For multi-intent / loose, expected_tool is None — show "→ {actual}" only - if s.get("expected_tool") is None: - arrow = actual - else: - arrow = f"{expected} → {actual}" - out += f"| {status} | `{sid}` | {metric} | {score_cell} | {arrow} |\n" - return out + "\n" - - -def render_failures(results: Dict[str, Any]) -> str: - failures = [ - s - for s in results.get("scenarios", []) - if not s.get("passed") and not s.get("error") - ] - if not failures: - return "" - out = "### Failed scenarios\n\n" - for s in failures: - sid = s.get("id", "?") - reason = s.get("reason") or "_(no reason captured)_" - preview = (s.get("final_response_preview") or "").replace("\n", " ") - if len(preview) > 200: - preview = preview[:200] + "…" - out += f"#### ❌ `{sid}` ({s.get('metric', '—')})\n\n" - out += f"- **Score:** {_score_str(s.get('score'), s.get('threshold', 0.0))}\n" - if s.get("expected_tool"): - out += f"- **Expected:** {_fmt_tool(s['expected_tool'])}\n" - out += f"- **Actual:** {_fmt_tool(s.get('actual_tool'))}\n" - out += f"- **Reason:** {reason}\n" - if preview: - out += f"- **Response preview:** `{preview}`\n" - out += "\n" - return out - - -def render_errors(results: Dict[str, Any]) -> str: - errored = [s for s in results.get("scenarios", []) if s.get("error")] - if not errored: - return "" - out = "### Errored scenarios (test harness errors, not assertion failures)\n\n" - for s in errored: - sid = s.get("id", "?") - err = s.get("error") or "_(no error captured)_" - out += f"- 💥 `{sid}` — {err}\n" - return out + "\n" - - -def render_methodology() -> str: - return ( - "### Methodology\n\n" - "- **Strict scenarios** (issue specifies the expected endpoint and " - "params) are scored with `ToolCorrectnessMetric`, " - "`evaluation_params=[ToolCallParams.INPUT_PARAMETERS]`, threshold 1.0. " - "Tool name must match exactly and every expected parameter must be " - "present with the expected value (extra parameters allowed).\n" - "- **Loose scenarios** (issue describes the flow but not the expected " - "resolution) are scored with `ArgumentCorrectnessMetric` (LLM-as-judge, " - "threshold 0.7).\n" - "- **Multi-intent scenarios** use `ArgumentCorrectnessMetric` when the " - "agent resolves a tool call; if the agent asks a clarifying question " - "instead, only routing-to-ATC is verified (rules out silent fall-through " - "to RAG/OOD).\n" - "- API tool endpoints are seeded into the testcontainers-backed Qdrant " - "from `tests/api_tool_eval/test-endpoints.json` via the " - "`api_tool_endpoints_indexed` fixture in `conftest.py`.\n\n" - ) - - -def render_report(results: Dict[str, Any]) -> str: - if results.get("error"): - return ( - f"## API Tool Calling Evaluation Report\n\n**ERROR:** {results['error']}\n" - ) - return ( - render_header(results) - + render_by_type(results) - + render_scenario_table(results) - + render_failures(results) - + render_errors(results) - + render_methodology() - ) - - -def main() -> int: - results = load_results() - markdown = render_report(results) - REPORT_FILE.write_text(markdown, encoding="utf-8") - print(f"Wrote {REPORT_FILE} ({len(markdown)} chars)") - if results.get("error"): - print(f" WARNING: {results['error']}", file=sys.stderr) - else: - print( - f" {results.get('passed_tests', 0)}/{results.get('total_tests', 0)} " - "scenarios passed" - ) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/deepeval_tests/api_tool_tests.py b/tests/deepeval_tests/api_tool_tests.py deleted file mode 100644 index 1edaae0..0000000 --- a/tests/deepeval_tests/api_tool_tests.py +++ /dev/null @@ -1,779 +0,0 @@ -""" -DeepEval tests for the API Tool Calling feature (issue #447). - -Covers the single-intent (Scenarios 1-4) and multi-intent (MI-1..MI-8) scenarios -listed in the issue against the running orchestration service. Each scenario -walks the ``/orchestrate`` endpoint turn-by-turn via the testcontainers-backed -``orchestration_client`` fixture, extracts the final agentic-loop tool call, -and scores it with a DeepEval agentic metric: - -* **Strict single-intent (S1, S2a, S2b, S3)** — the issue specifies an - "Expected endpoint" + URL/params per scenario. Scored with - ``ToolCorrectnessMetric`` (deterministic name + input-parameter comparison, - ``threshold=1.0``). - -* **Loose single-intent (S4)** — the issue documents the 5-turn flow but does - not specify an expected resolution. Scored with - ``ArgumentCorrectnessMetric`` (LLM-as-judge over the conversation input and - the resolved tool call). - -* **Multi-intent (MI-1..MI-8)** — issue lists only queries (EN+ET). If the - system resolves a tool call, scored with ``ArgumentCorrectnessMetric``; if - it asks a clarifying question instead, the test only verifies routing to - ATC (i.e. non-empty reply, not silent RAG/OOD). - -Endpoints are matched by ``name`` — the UUIDs in the issue do happen to match -those in ``tests/api_tool_eval/test-endpoints.json``, but name matching is the -stable contract. - -Depends on: -* ``orchestration_client`` — provides the testcontainers-mapped base URL. -* ``api_tool_endpoints_indexed`` — seeds the API tool fixture into Qdrant's - ``api_tool_collection`` so the agentic loop can find the endpoints. -""" - -import datetime -import json -import time -import uuid -from pathlib import Path -from typing import Any, Dict, List, Optional - -import pytest -import requests -from deepeval.metrics import ArgumentCorrectnessMetric, ToolCorrectnessMetric -from deepeval.test_case import LLMTestCase, ToolCall, ToolCallParams - -REQUEST_TIMEOUT = 60 -ENVIRONMENT = "development" -AUTHOR_ID = "api-tool-deepeval" - -# Where the result-collector writes the per-scenario record consumed by -# tests/deepeval_tests/api_tool_report_generator.py to render the PR -# comment / artifact markdown. -RESULTS_FILE = Path("api_tool_test_results.json") - -# Strict scenarios assert the exact expected tool was called with the exact -# expected params (extras allowed — see ToolCorrectnessMetric docs on -# should_exact_match). Threshold 1.0 because the deterministic comparison -# scores fractionally over expected_tools, and we want every expected param -# present and correct. -STRICT_TOOL_THRESHOLD = 1.0 - -# Loose scenarios are graded by an LLM judge — 0.7 matches the threshold used -# for the RAG metrics in standard_tests.py. -JUDGE_THRESHOLD = 0.7 - - -# --------------------------------------------------------------------------- -# HTTP helpers (mirror tests/api_tool_eval/integration_test_*.py) -# --------------------------------------------------------------------------- - - -def _make_chat_id(label: str) -> str: - return f"deepeval-api-tool-{label}-{uuid.uuid4().hex[:8]}" - - -def _send_turn( - base_url: str, - chat_id: str, - message: str, - history: List[Dict[str, str]], -) -> Dict[str, Any]: - payload: Dict[str, Any] = { - "chatId": chat_id, - "message": message, - "authorId": AUTHOR_ID, - "conversationHistory": history, - "url": "deepeval-test", - "environment": ENVIRONMENT, - } - resp = requests.post( - f"{base_url}/orchestrate", - json=payload, - timeout=REQUEST_TIMEOUT, - ) - resp.raise_for_status() - return resp.json() - - -def _append_history( - history: List[Dict[str, str]], - user_message: str, - bot_response: str, -) -> List[Dict[str, str]]: - ts = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) - return history + [ - {"authorRole": "user", "message": user_message, "timestamp": ts}, - {"authorRole": "bot", "message": bot_response, "timestamp": ts}, - ] - - -def _parse_completed(content: str) -> Optional[Dict[str, Any]]: - """Return the parsed JSON if content is a completed agentic-loop payload, - else None. A completed payload carries both ``endpoint`` and - ``collected_params`` keys.""" - try: - data = json.loads(content) - except (json.JSONDecodeError, TypeError): - return None - if "collected_params" in data and "endpoint" in data: - return data - return None - - -def _to_tool_call(content: str) -> Optional[ToolCall]: - """Build a DeepEval ToolCall from a completed agentic-loop payload, or - None if the response wasn't a completed JSON.""" - data = _parse_completed(content) - if data is None: - return None - ep = data.get("endpoint", {}) - name = ep.get("name", "") if isinstance(ep, dict) else str(ep) - params = data.get("collected_params", {}) or {} - return ToolCall(name=name, input_parameters=params) - - -def _conversation_text(turns: List[Dict[str, Any]]) -> str: - """Flatten the user turns into a single string for LLMTestCase.input. - - DeepEval's agentic single-turn metrics take a single ``input`` string; - this approximation gives the LLM judge the full conversational context. - """ - return "\n".join(f"USER: {t['user']}" for t in turns) - - -def _walk_turns(base_url: str, label: str, turns: List[Dict[str, Any]]) -> str: - """POST each turn in sequence, maintaining a stable chatId + history. - Returns the final bot response content.""" - chat_id = _make_chat_id(label) - history: List[Dict[str, str]] = [] - final_content = "" - for turn in turns: - resp = _send_turn(base_url, chat_id, turn["user"], history) - final_content = resp.get("content", "") - history = _append_history(history, turn["user"], final_content) - return final_content - - -def _tool_call_to_dict(tc: Optional[ToolCall]) -> Optional[Dict[str, Any]]: - """Serialize a ToolCall for the results JSON (None-tolerant).""" - if tc is None: - return None - return {"name": tc.name, "input_parameters": dict(tc.input_parameters or {})} - - -# --------------------------------------------------------------------------- -# Result collector — every test pushes one record, autouse fixture flushes to -# disk at session end. tests/deepeval_tests/api_tool_report_generator.py -# reads the JSON and renders the markdown report consumed by the workflow. -# --------------------------------------------------------------------------- - - -class ApiToolResultCollector: - """Accumulates per-scenario results from the API tool tests.""" - - def __init__(self) -> None: - self.results: Dict[str, Any] = { - "total_tests": 0, - "passed_tests": 0, - "failed_tests": 0, - "errored_tests": 0, - "test_start_time": datetime.datetime.now().isoformat(), - "scenarios": [], - } - - def add( - self, - scenario_id: str, - scenario_type: str, - metric_name: str, - threshold: float, - score: Optional[float], - passed: bool, - reason: str = "", - error: str = "", - expected_tool: Optional[Dict[str, Any]] = None, - actual_tool: Optional[Dict[str, Any]] = None, - final_response_preview: str = "", - extra: Optional[Dict[str, Any]] = None, - ) -> None: - self.results["total_tests"] += 1 - if error: - self.results["errored_tests"] += 1 - elif passed: - self.results["passed_tests"] += 1 - else: - self.results["failed_tests"] += 1 - self.results["scenarios"].append( - { - "id": scenario_id, - "type": scenario_type, - "metric": metric_name, - "threshold": threshold, - "score": score, - "passed": passed, - "reason": reason, - "error": error, - "expected_tool": expected_tool, - "actual_tool": actual_tool, - "final_response_preview": final_response_preview, - "extra": extra or {}, - } - ) - - def save(self, path: Path = RESULTS_FILE) -> None: - with open(path, "w", encoding="utf-8") as f: - json.dump(self.results, f, indent=2, default=str, ensure_ascii=False) - print( - f"Saved API tool results to {path}: " - f"{self.results['passed_tests']}/{self.results['total_tests']} passed, " - f"{self.results['failed_tests']} failed, " - f"{self.results['errored_tests']} errored" - ) - - -_collector = ApiToolResultCollector() - - -@pytest.fixture(scope="session", autouse=True) -def _save_api_tool_results(): - """Flush collected results to RESULTS_FILE at end of session, even on - failure — mirrors save_results_fixture in standard_tests.py.""" - yield - _collector.save() - - -# --------------------------------------------------------------------------- -# Strict single-intent scenarios — issue specifies expected endpoint + params -# (S1, S2a, S2b, S3). Scored with ToolCorrectnessMetric. -# --------------------------------------------------------------------------- - - -STRICT_SINGLE_INTENT_SCENARIOS: List[Dict[str, Any]] = [ - # Scenario 1 — Normal Workflow (citizen initiative details) - { - "id": "S1-citizen-initiative-EN", - "label": "s1-en", - "turns": [ - {"user": "Can I see the details of a citizen initiative?"}, - {"user": "1790"}, - ], - "expected_tool": ToolCall( - name="get_initiative_details", - input_parameters={"id": "1790"}, - ), - }, - { - "id": "S1-citizen-initiative-ET", - "label": "s1-et", - "turns": [ - {"user": "Kas ma saan kodanikualgatuse üksikasju vaadata?"}, - {"user": "1790"}, - ], - "expected_tool": ToolCall( - name="get_initiative_details", - input_parameters={"id": "1790"}, - ), - }, - # Scenario 2a — Public Holidays (date range correction) - { - "id": "S2a-public-holidays-date-correction-EN", - "label": "s2a-en", - "turns": [ - {"user": "What are the public holidays in Estonia?"}, - {"user": "From 2026-01-01"}, - { - "user": ( - "My mistake — the correct period is April 1, 2026 " - "through December 31, 2026." - ) - }, - ], - "expected_tool": ToolCall( - name="get_public_holidays", - input_parameters={ - "countryIsoCode": "EE", - "validFrom": "2026-04-01", - "validTo": "2026-12-31", - }, - ), - }, - { - "id": "S2a-public-holidays-date-correction-ET", - "label": "s2a-et", - "turns": [ - {"user": "Millised on riigipühad Eestis?"}, - {"user": "Alates 1. jaanuarist 2026"}, - { - "user": ( - "Minu viga, õige periood on 01.04.2026 kuni 31.12.2026. " - "Tegelikult tahan 2026-04-01 kuni 2026-12-31." - ) - }, - ], - "expected_tool": ToolCall( - name="get_public_holidays", - input_parameters={ - "countryIsoCode": "EE", - "validFrom": "2026-04-01", - "validTo": "2026-12-31", - }, - ), - }, - # Scenario 2b — Parliament Votings (date range correction) - { - "id": "S2b-parliament-votings-date-correction-EN", - "label": "s2b-en", - "turns": [ - {"user": "What votes took place in the Estonian parliament?"}, - {"user": "2026-04-05"}, - { - "user": ( - "My mistake — the correct period is April 6, 2026 " - "through April 7, 2026." - ) - }, - ], - "expected_tool": ToolCall( - name="get_parliament_votings", - input_parameters={ - "startDate": "2026-04-06", - "endDate": "2026-04-07", - }, - ), - }, - { - "id": "S2b-parliament-votings-date-correction-DE", - "label": "s2b-de", - "turns": [ - {"user": ("Welche Abstimmungen fanden im estnischen Parlament statt?")}, - {"user": "2026-04-05"}, - { - "user": ( - "Mein Fehler — der richtige Zeitraum ist vom " - "6. April 2026 bis zum 7. April 2026." - ) - }, - ], - "expected_tool": ToolCall( - name="get_parliament_votings", - input_parameters={ - "startDate": "2026-04-06", - "endDate": "2026-04-07", - }, - ), - }, - # Scenario 3 — Intent Switch (electricity prices -> address search) - { - "id": "S3-intent-switch-EN", - "label": "s3-en", - "turns": [ - {"user": "Show last week's electricity prices in Estonia."}, - { - "user": ( - "Wait — could you check the following location instead: " - "Viru tn 4, Tallinn?" - ) - }, - ], - "expected_tool": ToolCall( - name="search_address", - input_parameters={"address": "Viru tn 4, Tallinn"}, - ), - }, - { - "id": "S3-intent-switch-ET", - "label": "s3-et", - "turns": [ - {"user": "Näita eelmise nädala elektrienergia hindu Eestis."}, - { - "user": ( - "Oota, kas saaksid hoopis järgmist asukohta kontrollida: " - "Viru tn 4, Tallinn?" - ) - }, - ], - "expected_tool": ToolCall( - name="search_address", - input_parameters={"address": "Viru tn 4, Tallinn"}, - ), - }, -] - - -@pytest.mark.parametrize( - "scenario", - STRICT_SINGLE_INTENT_SCENARIOS, - ids=[s["id"] for s in STRICT_SINGLE_INTENT_SCENARIOS], -) -def test_api_tool_strict_single_intent( - scenario: Dict[str, Any], - orchestration_client: Any, - api_tool_endpoints_indexed: None, -) -> None: - """Scenarios 1, 2a, 2b, 3 — issue specifies the expected tool call. - - Scored deterministically with ``ToolCorrectnessMetric``: - * the resolved tool's name must equal the expected name, and - * every expected input parameter must be present and equal in the - ``collected_params`` (extras allowed). - """ - del api_tool_endpoints_indexed # consumed for its setup side effect only - - expected_tool: ToolCall = scenario["expected_tool"] - final_content = "" - actual_tool: Optional[ToolCall] = None - score: Optional[float] = None - reason = "" - passed = False - error = "" - - try: - final_content = _walk_turns( - orchestration_client.base_url, scenario["label"], scenario["turns"] - ) - actual_tool = _to_tool_call(final_content) - test_case = LLMTestCase( - input=_conversation_text(scenario["turns"]), - actual_output=final_content, - tools_called=[actual_tool] if actual_tool is not None else [], - expected_tools=[expected_tool], - ) - metric = ToolCorrectnessMetric( - threshold=STRICT_TOOL_THRESHOLD, - evaluation_params=[ToolCallParams.INPUT_PARAMETERS], - ) - metric.measure(test_case) - score = metric.score - reason = metric.reason or "" - passed = score is not None and score >= STRICT_TOOL_THRESHOLD - assert passed, ( - f"[{scenario['id']}] tool correctness {score} < " - f"{STRICT_TOOL_THRESHOLD}: {reason}\n" - f"Expected tool: {expected_tool.name}({expected_tool.input_parameters})\n" - f"Final response: {final_content[:300]}" - ) - except AssertionError: - raise - except Exception as e: - error = f"{type(e).__name__}: {e}" - raise - finally: - _collector.add( - scenario_id=scenario["id"], - scenario_type="strict", - metric_name="ToolCorrectnessMetric", - threshold=STRICT_TOOL_THRESHOLD, - score=score, - passed=passed, - reason=reason, - error=error, - expected_tool=_tool_call_to_dict(expected_tool), - actual_tool=_tool_call_to_dict(actual_tool), - final_response_preview=final_content[:300], - ) - - -# --------------------------------------------------------------------------- -# Loose single-intent scenarios — issue documents the flow but does not -# specify an "Expected endpoint" (S4, both languages). Scored with -# ArgumentCorrectnessMetric (LLM judges arg correctness vs. the -# conversation input). -# --------------------------------------------------------------------------- - - -LOOSE_SINGLE_INTENT_SCENARIOS: List[Dict[str, Any]] = [ - { - "id": "S4-parliament-attendance-multi-turn-EN", - "label": "s4-en", - "turns": [ - { - "user": ( - "Can you show me the parliament attendance of former " - "Finance Minister Martin Helme?" - ) - }, - {"user": "Can you just check with what you have?"}, - {"user": "2026-04-01"}, - {"user": "Yes"}, - {"user": "2026-04-20"}, - ], - }, - { - "id": "S4-parliament-attendance-multi-turn-ET", - "label": "s4-et", - "turns": [ - { - "user": ( - "Kas saaksite mulle näidata endise rahandusministri " - "Martin Helme parlamendi kohaloleku andmeid?" - ) - }, - {"user": "Kas saaksite lihtsalt oma andmetest järele vaadata?"}, - {"user": "2026-04-01"}, - {"user": "Jah"}, - {"user": "2026-04-20"}, - ], - }, -] - - -@pytest.mark.parametrize( - "scenario", - LOOSE_SINGLE_INTENT_SCENARIOS, - ids=[s["id"] for s in LOOSE_SINGLE_INTENT_SCENARIOS], -) -def test_api_tool_loose_single_intent( - scenario: Dict[str, Any], - orchestration_client: Any, - api_tool_endpoints_indexed: None, -) -> None: - """Scenario 4 (EN/ET) — issue gives no expected resolution. - - Asserts: - 1. The final turn produced a completed JSON tool call (i.e. the agentic - loop actually resolved, didn't fall through to RAG). - 2. The LLM judge ``ArgumentCorrectnessMetric`` is satisfied that the - chosen tool's arguments fit the conversation input. - """ - del api_tool_endpoints_indexed # consumed for its setup side effect only - - final_content = "" - actual_tool: Optional[ToolCall] = None - score: Optional[float] = None - reason = "" - passed = False - error = "" - - try: - final_content = _walk_turns( - orchestration_client.base_url, scenario["label"], scenario["turns"] - ) - actual_tool = _to_tool_call(final_content) - - assert actual_tool is not None, ( - f"[{scenario['id']}] expected a completed JSON tool call on the " - f"final turn but got: {final_content[:300]}" - ) - - test_case = LLMTestCase( - input=_conversation_text(scenario["turns"]), - actual_output=final_content, - tools_called=[actual_tool], - ) - metric = ArgumentCorrectnessMetric(threshold=JUDGE_THRESHOLD) - metric.measure(test_case) - score = metric.score - reason = metric.reason or "" - passed = score is not None and score >= JUDGE_THRESHOLD - assert passed, ( - f"[{scenario['id']}] argument correctness {score} < " - f"{JUDGE_THRESHOLD}: {reason}\n" - f"Resolved tool: {actual_tool.name}({actual_tool.input_parameters})" - ) - except AssertionError: - raise - except Exception as e: - error = f"{type(e).__name__}: {e}" - raise - finally: - _collector.add( - scenario_id=scenario["id"], - scenario_type="loose", - metric_name="ArgumentCorrectnessMetric", - threshold=JUDGE_THRESHOLD, - score=score, - passed=passed, - reason=reason, - error=error, - actual_tool=_tool_call_to_dict(actual_tool), - final_response_preview=final_content[:300], - ) - - -# --------------------------------------------------------------------------- -# Multi-Intent scenarios (issue #447, MI-1..MI-8) -# -# Issue lists 8 queries (EN + ET) with no expected resolution. Under Phase 1 -# the orchestrator decomposes the query and falls back to a single endpoint -# (see tests/api_tool_eval/integration_test_multi_intent.py docstring). We: -# -# * If the system resolves a tool call → score with ArgumentCorrectnessMetric -# (LLM judges whether the chosen args fit the multi-intent query). -# * If the system asks a clarifying question → accept it (still ATC-routed), -# only assert the reply is non-empty (rules out a silent RAG/OOD fallthrough). -# --------------------------------------------------------------------------- - - -MULTI_INTENT_SCENARIOS: List[Dict[str, Any]] = [ - { - "id": "MI-1-address-and-vehicle-tax", - "label": "mi1", - "query_en": ( - "Can you find an address for me and also calculate my vehicle " - "tax? (Address: Viru tn 4, Tallinn / Plate: 123ABC / Year: 2026)" - ), - "query_et": ( - "Kas saaksite mulle aadressi leida ja arvutada ka mu sõiduki " - "maksu? (Aadress: Viru tn 4, Tallinn / Registreerimismärk: " - "123ABC / Aasta: 2026)" - ), - }, - { - "id": "MI-2-address-and-initiative-details", - "label": "mi2", - "query_en": ( - "I need to find an address and also check details of an " - "initiative. (Address: Viru tn 4, Tallinn / Initiative ID: 1790)" - ), - "query_et": ( - "Mul on vaja leida aadress ja vaadata ka kodanikualgatuse " - "üksikasju. (Aadress: Viru tn 4, Tallinn / Algatuse ID: 1790)" - ), - }, - { - "id": "MI-3-electricity-and-public-holidays", - "label": "mi3", - "query_en": ( - "Show me electricity prices in Estonia and list Estonia's public holidays." - ), - "query_et": ("Näita mulle Eesti elektrihindu ja too välja Eesti riigipühad."), - }, - { - "id": "MI-4-parliament-votings-and-initiatives", - "label": "mi4", - "query_en": ( - "Show me the parliament voting results and also list the " - "citizen initiatives." - ), - "query_et": ( - "Näita mulle parlamendi hääletustulemusi ja too välja ka kodanikualgatused." - ), - }, - { - "id": "MI-5-address-and-parliament-participation", - "label": "mi5", - "query_en": ( - "Could you find me an address and also show the attendance " - "statistics of Riigikogu members?" - ), - "query_et": ( - "Kas saaksite mulle leida aadressi ja näidata ka Riigikogu " - "liikmete osalusstatistikat?" - ), - }, - { - "id": "MI-6-public-holidays-and-vehicle-tax", - "label": "mi6", - "query_en": ( - "What are the public holidays in Estonia and can you also " - "calculate my vehicle tax?" - ), - "query_et": ( - "Millised on riigipühad Eestis ja kas saate arvutada ka mu sõiduki maksu?" - ), - }, - { - "id": "MI-7-initiatives-and-parliament-votings", - "label": "mi7", - "query_en": ( - "Show me all citizen initiatives and also display the parliament " - "voting results." - ), - "query_et": ( - "Kuva mulle kõik kodanikualgatused ja näita ka riigikogu hääletustulemusi." - ), - }, - { - "id": "MI-8-vehicle-tax-and-electricity", - "label": "mi8", - "query_en": ( - "Calculate my vehicle tax and also show me the electricity " - "market price in Estonia." - ), - "query_et": ( - "Arvuta mu sõiduki maks ja näita mulle ka Eesti elektri turuhinda." - ), - }, -] - - -@pytest.mark.parametrize( - "scenario", - MULTI_INTENT_SCENARIOS, - ids=[s["id"] for s in MULTI_INTENT_SCENARIOS], -) -@pytest.mark.parametrize("lang", ["en", "et"]) -def test_api_tool_multi_intent( - scenario: Dict[str, Any], - lang: str, - orchestration_client: Any, - api_tool_endpoints_indexed: None, -) -> None: - del api_tool_endpoints_indexed # consumed for its setup side effect only - - query = scenario[f"query_{lang}"] - content = "" - actual_tool: Optional[ToolCall] = None - score: Optional[float] = None - reason = "" - passed = False - error = "" - outcome = "unknown" # "tool_call" | "clarifying_question" - - try: - base_url = orchestration_client.base_url - chat_id = _make_chat_id(f"{scenario['label']}-{lang}") - resp = _send_turn(base_url, chat_id, query, []) - content = resp.get("content", "") - actual_tool = _to_tool_call(content) - - if actual_tool is not None: - outcome = "tool_call" - test_case = LLMTestCase( - input=query, - actual_output=content, - tools_called=[actual_tool], - ) - metric = ArgumentCorrectnessMetric(threshold=JUDGE_THRESHOLD) - metric.measure(test_case) - score = metric.score - reason = metric.reason or "" - passed = score is not None and score >= JUDGE_THRESHOLD - assert passed, ( - f"[{scenario['id']} {lang}] argument correctness {score} " - f"< {JUDGE_THRESHOLD}: {reason}\n" - f"Resolved tool: {actual_tool.name}({actual_tool.input_parameters})" - ) - else: - outcome = "clarifying_question" - # No tool call yet — must at least be a non-empty clarifying reply - # (rules out silent failure / RAG fallthrough returning no content). - passed = bool(content.strip()) - assert passed, ( - f"[{scenario['id']} {lang}] empty response — multi-intent query " - f"failed routing entirely. Got: {content!r}" - ) - reason = "Resolved to a clarifying question (no tool call yet)" - except AssertionError: - raise - except Exception as e: - error = f"{type(e).__name__}: {e}" - raise - finally: - _collector.add( - scenario_id=f"{scenario['id']}-{lang}", - scenario_type="multi_intent", - metric_name=( - "ArgumentCorrectnessMetric" if outcome == "tool_call" else "RoutingOnly" - ), - threshold=JUDGE_THRESHOLD if outcome == "tool_call" else 0.0, - score=score, - passed=passed, - reason=reason, - error=error, - actual_tool=_tool_call_to_dict(actual_tool), - final_response_preview=content[:300], - extra={"language": lang, "outcome": outcome, "query": query}, - ) diff --git a/tests/deepeval_tests/conftest.py b/tests/deepeval_tests/conftest.py index 7bb40f2..b598a35 100644 --- a/tests/deepeval_tests/conftest.py +++ b/tests/deepeval_tests/conftest.py @@ -796,106 +796,3 @@ def __init__(self, base_url: str): self.base_url = base_url return OrchestrationClient(rag_stack.get_orchestration_service_url()) - - -@pytest.fixture(scope="session") -def api_tool_endpoints_indexed(rag_stack: RAGStackTestContainers): - """ - Session-scoped fixture that seeds the API tool endpoints from - ``tests/api_tool_eval/test-endpoints.json`` into Qdrant's - ``api_tool_collection`` so the agentic loop can find them. - - Required by tests in ``tests/deepeval_tests/api_tool_tests.py`` — without - it, semantic search over API tools returns nothing and the orchestrator - falls back to RAG. - - The indexer module hard-codes container-internal URLs in its constants - (``http://llm-orchestration-service:8100``, ``qdrant:6333``). Two of the - three are used in code paths that re-read the constant at call time - (monkey-patching the class attribute works), but ``ApiToolQdrantManager`` - is instantiated with no arguments inside ``index_endpoint`` and binds the - host/port at function-definition time as defaults. To override those, the - fixture also replaces the ``ApiToolQdrantManager`` name imported into - ``main_indexer`` with a factory that injects the testcontainers-mapped - host/port. - """ - import asyncio - import json as _json - from pathlib import Path as _Path - from unittest.mock import patch - from urllib.parse import urlparse - - from api_tool_indexer import main_indexer - from api_tool_indexer.constants import ApiToolIndexerConstants - from api_tool_indexer.main_indexer import index_endpoint - from api_tool_indexer.models import EndpointData - from api_tool_indexer.qdrant_manager import ApiToolQdrantManager - - orch_url = rag_stack.get_orchestration_service_url() - qdrant_url = rag_stack.get_qdrant_url() - parsed = urlparse(qdrant_url) - qdrant_host = parsed.hostname or "localhost" - qdrant_port = parsed.port or 6333 - - def _qdrant_factory(*args: Any, **kwargs: Any) -> ApiToolQdrantManager: - kwargs.setdefault("host", qdrant_host) - kwargs.setdefault("port", qdrant_port) - return ApiToolQdrantManager(*args, **kwargs) - - endpoints_file = ( - _Path(__file__).parent.parent / "api_tool_eval" / "test-endpoints.json" - ) - if not endpoints_file.exists(): - pytest.skip( - f"API tool endpoint fixture not found: {endpoints_file} — " - "cannot seed api_tool_collection" - ) - - with open(endpoints_file, encoding="utf-8") as f: - endpoints = _json.load(f) - - async def _seed_all() -> list: - results = [] - for ep in endpoints: - endpoint_data = EndpointData( - endpoint_id=ep["endpointId"], - name=ep["name"], - description=ep["description"], - url=ep["url"], - method=ep["method"], - params=ep.get("params", []), - ) - res = await index_endpoint(endpoint_data) - logger.info( - f"Seeded endpoint '{ep['name']}': success={res.success} ({res.message})" - ) - results.append((ep["name"], res)) - return results - - # Connection ID "evalconnection-1" matches the one used by standard_tests.py - # and the EVAL_MODE setup; vault is seeded for it. The default - # "gpt-4o-mini" used by the indexer in production has no test fixture. - with ( - patch.object(ApiToolIndexerConstants, "DEFAULT_API_BASE_URL", orch_url), - patch.object( - ApiToolIndexerConstants, - "DEFAULT_CONNECTION_ID", - "evalconnection-1", - ), - patch.object(main_indexer, "ApiToolQdrantManager", _qdrant_factory), - ): - logger.info( - f"Seeding {len(endpoints)} API tool endpoints " - f"(orch={orch_url}, qdrant={qdrant_host}:{qdrant_port})" - ) - results = asyncio.run(_seed_all()) - - failed = [name for name, r in results if not r.success] - if failed: - pytest.skip( - f"API tool seeding failed for {len(failed)}/{len(endpoints)} " - f"endpoints ({failed}) — skipping API tool tests" - ) - - logger.info(f"Indexed all {len(endpoints)} API tool endpoints successfully") - yield From 8edf65d2e7d525f9bf6e2d80679b6d24e7d7beab Mon Sep 17 00:00:00 2001 From: nuwangeek Date: Fri, 26 Jun 2026 09:10:36 +0530 Subject: [PATCH 5/5] fixed issue --- tests/deepeval_tests/conftest.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/deepeval_tests/conftest.py b/tests/deepeval_tests/conftest.py index b598a35..40ba463 100644 --- a/tests/deepeval_tests/conftest.py +++ b/tests/deepeval_tests/conftest.py @@ -384,11 +384,11 @@ def _write_test_secrets(self, client: hvac.Client) -> None: client.secrets.kv.v2.create_or_update_secret( mount_point="secret", - path="llm/connections/azure_openai/development/evalconnection-1", + path="llm/connections/azure_openai/evalconnection-1", secret=llm_secret, ) logger.info( - "LLM connection secret written to llm/connections/azure_openai/development/evalconnection-1" + "LLM connection secret written to llm/connections/azure_openai/evalconnection-1" ) # ============================================================ @@ -413,17 +413,17 @@ def _write_test_secrets(self, client: hvac.Client) -> None: logger.info(f" → model: {embedding_secret['model']}") logger.info(f" → connection_id: {embedding_secret['connection_id']}") logger.info( - " → Vault path: embeddings/connections/azure_openai/development/evalconnection-1" + " → Vault path: embeddings/connections/azure_openai/evalconnection-1" ) # Write to embeddings path with connection_id in the path client.secrets.kv.v2.create_or_update_secret( mount_point="secret", - path="embeddings/connections/azure_openai/development/evalconnection-1", + path="embeddings/connections/azure_openai/evalconnection-1", secret=embedding_secret, ) logger.info( - "Embedding secret written to embeddings/connections/azure_openai/development/evalconnection-1" + "Embedding secret written to embeddings/connections/azure_openai/evalconnection-1" ) # ============================================================ @@ -434,7 +434,7 @@ def _write_test_secrets(self, client: hvac.Client) -> None: try: # Verify LLM path verify_llm = client.secrets.kv.v2.read_secret_version( - path="llm/connections/azure_openai/development/evalconnection-1", + path="llm/connections/azure_openai/evalconnection-1", mount_point="secret", ) llm_data = verify_llm["data"]["data"] @@ -443,7 +443,7 @@ def _write_test_secrets(self, client: hvac.Client) -> None: # Verify embeddings path verify_embedding = client.secrets.kv.v2.read_secret_version( - path="embeddings/connections/azure_openai/development/evalconnection-1", + path="embeddings/connections/azure_openai/evalconnection-1", mount_point="secret", ) embedding_data = verify_embedding["data"]["data"] @@ -617,7 +617,7 @@ def _verify_token_permissions(self, client: hvac.Client) -> None: """Verify the token has correct permissions to read secrets""" try: client.secrets.kv.v2.read_secret_version( - path="llm/connections/azure_openai/development/evalconnection-1", + path="llm/connections/azure_openai/evalconnection-1", mount_point="secret", ) logger.info("Token has correct permissions to read secrets")