From 1fc3b9c967a950c3612600f6b4db0428aaabddce Mon Sep 17 00:00:00 2001 From: nuwangeek Date: Fri, 17 Apr 2026 12:40:20 +0530 Subject: [PATCH 1/6] standalone agentic loop module --- src/tool_classifier/__init__.py | 12 +- src/tool_classifier/agentic_loop.py | 326 ++++++++++ src/tool_classifier/constants.py | 26 + src/tool_classifier/context_analyzer.py | 4 +- src/tool_classifier/enums.py | 20 + src/tool_classifier/models.py | 24 +- tests/test_agentic_loop.py | 781 ++++++++++++++++++++++++ 7 files changed, 1186 insertions(+), 7 deletions(-) create mode 100644 src/tool_classifier/agentic_loop.py create mode 100644 tests/test_agentic_loop.py diff --git a/src/tool_classifier/__init__.py b/src/tool_classifier/__init__.py index 38b861d5..7e7e7c96 100644 --- a/src/tool_classifier/__init__.py +++ b/src/tool_classifier/__init__.py @@ -9,12 +9,16 @@ - Layer 4: OOD Workflow (out-of-domain fallback) """ -from .classifier import ToolClassifier -from .enums import WorkflowType -from .models import ClassificationResult +from tool_classifier.agentic_loop import AgenticLoop +from tool_classifier.classifier import ToolClassifier +from tool_classifier.enums import AgenticLoopStatus, WorkflowType +from tool_classifier.models import AgenticLoopResult, ClassificationResult __all__ = [ + "AgenticLoop", + "AgenticLoopResult", + "AgenticLoopStatus", + "ClassificationResult", "ToolClassifier", "WorkflowType", - "ClassificationResult", ] diff --git a/src/tool_classifier/agentic_loop.py b/src/tool_classifier/agentic_loop.py new file mode 100644 index 00000000..08915b75 --- /dev/null +++ b/src/tool_classifier/agentic_loop.py @@ -0,0 +1,326 @@ +"""Standalone agentic loop for multi-turn parameter collection.""" + +import asyncio +from typing import Any, Dict, List + +from loguru import logger + +from src.utils.api_tool_session_store import APIToolSessionStore +from tool_classifier.constants import CONTINUATION_QUESTION, CONTINUATION_TURN +from tool_classifier.enums import AgenticLoopStatus +from tool_classifier.models import AgenticLoopResult +from tool_classifier.param_extractor import ParamExtractionModule + +# Normalised user responses that indicate the user wants to keep collecting params. +_YES_RESPONSES = frozenset( + { + "yes", + "y", + "jah", + "ja", + "да", + "ok", + "okay", + "sure", + "please", + "continue", + "jätka", + "продолжить", + "absolutely", + } +) + + +class AgenticLoop: + """Stateless multi-turn parameter collection loop. + + Each call to run_turn() represents one user message / one loop iteration. + The loop carries no internal state — all state is passed in as arguments. + Redis persistence (load from session before calling, save inside run_turn) + is handled here so callers only need to act on the returned AgenticLoopResult. + + Typical usage:: + + loop = AgenticLoop( + session_store=app.state.session_store, + param_extractor=ParamExtractionModule(), + ) + + result = await loop.run_turn( + chat_id=request.chatId, + user_message=request.message, + conversation_history=request.conversationHistory, + params_schema=endpoint["params_schema"], + collected_params=session.collected_params, + turn_count=session.turn_count, + max_turns=session.max_turns, + ) + + if result.status == AgenticLoopStatus.COMPLETED: + # All params ready — call the API, then delete session + ... + elif result.status == AgenticLoopStatus.NEEDS_INPUT: + # Session already saved inside run_turn — return question to user + ... + else: # MAX_TURNS_REACHED + # Delete session and fall back gracefully + ... + """ + + def __init__( + self, + session_store: APIToolSessionStore, + param_extractor: ParamExtractionModule, + ) -> None: + """Initialise the loop with an injected session store and param extractor. + + Args: + session_store: Redis-backed store used to persist loop state between + HTTP requests. Injected to allow easy mocking in tests. + param_extractor: DSPy module that extracts parameter values from a + user message. Injected to allow easy mocking in tests. + """ + self._session_store = session_store + self._param_extractor = param_extractor + + async def run_turn( + self, + chat_id: str, + user_message: str, + conversation_history: List[Dict[str, Any]], + params_schema: List[Dict[str, Any]], + collected_params: Dict[str, Any], + turn_count: int, + max_turns: int = 5, + awaiting_continuation: bool = False, + continuation_turn: int = CONTINUATION_TURN, + ) -> AgenticLoopResult: + """Process one user turn of the parameter-collection loop. + + Steps: + 0. Continuation decision — if ``awaiting_continuation`` is True, detect + whether the user said yes (keep going) or no (fall back to RAG). + A "no" or ambiguous response returns MAX_TURNS_REACHED immediately. + 1. Guard — return MAX_TURNS_REACHED if the turn limit is reached. + 2. Extract — call ParamExtractionModule for newly mentioned params. + 3. Merge — combine prior collected params with newly extracted ones. + Prior values are authoritative (not overwritten by this turn). + 4. Completeness check — if all required params are present, save state + and return COMPLETED. + 5. Incomplete — if this is exactly the ``continuation_turn``, save state + and return AWAITING_CONTINUATION_DECISION with a yes/no question. + Otherwise return NEEDS_INPUT with the clarifying question. + + The returned turn_count is always input turn_count + 1. + Session state is saved automatically on COMPLETED, NEEDS_INPUT, and + AWAITING_CONTINUATION_DECISION. + It is NOT saved on MAX_TURNS_REACHED or extraction errors — the caller + is expected to delete the session in those cases. + + Args: + chat_id: Unique conversation identifier, used as the Redis session key. + user_message: The user's latest message for this turn. + conversation_history: Recent conversation turns as a list of + ``{"authorRole": str, "message": str}`` dicts. + params_schema: Parameter schema defining what to collect. Each + entry is a dict with at minimum ``name``, ``type``, + ``required``, and ``description`` keys. + collected_params: Parameter values collected in prior turns. + These are treated as authoritative and will not be overwritten. + turn_count: The current turn index (0-based before this call). + max_turns: Maximum turns allowed before the loop is abandoned. + awaiting_continuation: True when the previous turn returned + AWAITING_CONTINUATION_DECISION and we are now processing the + user's yes/no reply. Load this from the persisted session. + continuation_turn: The 1-based turn count at which to ask the + continuation question when params are still missing. + Defaults to ``CONTINUATION_TURN`` (3). + + Returns: + AgenticLoopResult with updated status, collected_params, and + turn_count. + """ + updated_turn_count = turn_count + 1 + + # Step 0 — Continuation decision: user is responding to the yes/no prompt + original_awaiting_continuation = awaiting_continuation + if awaiting_continuation: + wants_to_continue = self._detect_continuation_response(user_message) + if wants_to_continue: + logger.debug( + "AgenticLoop: user chose to continue on turn {} for chat_id={}", + turn_count, + chat_id, + ) + # Reset the flag so normal extraction takes over from here. + awaiting_continuation = False + else: + logger.info( + "AgenticLoop: user chose to exit on turn {} for chat_id={}, " + "falling back to RAG", + turn_count, + chat_id, + ) + return AgenticLoopResult( + status=AgenticLoopStatus.MAX_TURNS_REACHED, + collected_params=collected_params, + clarifying_question="", + turn_count=updated_turn_count, + ) + + # Step 1 — Turn limit guard (no session save — caller deletes) + if turn_count >= max_turns: + logger.warning( + "AgenticLoop: max_turns={} reached for chat_id={}, abandoning", + max_turns, + chat_id, + ) + return AgenticLoopResult( + status=AgenticLoopStatus.MAX_TURNS_REACHED, + collected_params=collected_params, + clarifying_question="", + turn_count=updated_turn_count, + ) + + # Step 2 — Extract params from the current user message + try: + extraction = await asyncio.to_thread( + self._param_extractor, + user_message, + params_schema, + conversation_history, + collected_params, + ) + except Exception as exc: + logger.error( + "AgenticLoop: param extraction failed on turn {} for chat_id={}: {}", + turn_count, + chat_id, + exc, + ) + # If a continuation decision was already consumed this turn, persist the + # updated flag so the next user message is not misread as another + # yes/no continuation response. + if awaiting_continuation != original_awaiting_continuation: + await self._save_session( + chat_id, + collected_params, + updated_turn_count, + awaiting_continuation=awaiting_continuation, + ) + return AgenticLoopResult( + status=AgenticLoopStatus.NEEDS_INPUT, + collected_params=collected_params, + clarifying_question="", + turn_count=updated_turn_count, + ) + + # Step 3 — Merge: prior values take precedence (already_collected is authoritative) + merged_params: Dict[str, Any] = { + **extraction["extracted_params"], + **collected_params, + } + + # Step 4 — Completeness check + required_param_names = { + p["name"] + for p in params_schema + if isinstance(p, dict) and p.get("required", False) + } + all_collected = required_param_names.issubset(merged_params.keys()) + + if all_collected: + logger.debug( + "AgenticLoop: all required params collected on turn {} for chat_id={}", + turn_count, + chat_id, + ) + await self._save_session( + chat_id, merged_params, updated_turn_count, awaiting_continuation=False + ) + return AgenticLoopResult( + status=AgenticLoopStatus.COMPLETED, + collected_params=merged_params, + clarifying_question="", + turn_count=updated_turn_count, + ) + + # Step 5 — Still missing params + logger.debug( + "AgenticLoop: turn {} for chat_id={} — still missing: {}", + turn_count, + chat_id, + extraction["missing_required"], + ) + + # At exactly the continuation threshold, ask whether to keep going. + if updated_turn_count == continuation_turn: + logger.info( + "AgenticLoop: continuation threshold reached on turn {} for chat_id={}", + turn_count, + chat_id, + ) + await self._save_session( + chat_id, merged_params, updated_turn_count, awaiting_continuation=True + ) + return AgenticLoopResult( + status=AgenticLoopStatus.AWAITING_CONTINUATION_DECISION, + collected_params=merged_params, + clarifying_question=CONTINUATION_QUESTION, + turn_count=updated_turn_count, + ) + + await self._save_session( + chat_id, merged_params, updated_turn_count, awaiting_continuation=False + ) + return AgenticLoopResult( + status=AgenticLoopStatus.NEEDS_INPUT, + collected_params=merged_params, + clarifying_question=extraction["clarifying_question"], + turn_count=updated_turn_count, + ) + + async def _save_session( + self, + chat_id: str, + collected_params: Dict[str, Any], + turn_count: int, + awaiting_continuation: bool = False, + ) -> None: + """Persist updated loop state to the Redis session store. + + Only updates the fields the loop owns (collected_params, turn_count, + awaiting_continuation). Workflow-owned fields (selected_endpoint, + state, max_turns) are preserved. + A missing or unavailable session is logged but never raises. + """ + try: + await self._session_store.update( + chat_id, + collected_params=collected_params, + turn_count=turn_count, + awaiting_continuation=awaiting_continuation, + ) + except Exception as exc: + logger.error( + "AgenticLoop: failed to save session for chat_id={}: {}", + chat_id, + exc, + ) + + def _detect_continuation_response(self, user_message: str) -> bool: + """Detect whether the user's message indicates they want to continue. + + Checks the normalised (lower-cased, stripped) message against a set of + known affirmative responses in Estonian, English, and Russian. + Any response that is not clearly affirmative is treated as a "no" so + the loop falls back to the RAG workflow. + + Args: + user_message: The raw user message to inspect. + + Returns: + True if the user wants to continue, False otherwise. + """ + normalised = user_message.strip().lower() + return normalised in _YES_RESPONSES diff --git a/src/tool_classifier/constants.py b/src/tool_classifier/constants.py index 99284cd1..9628a921 100644 --- a/src/tool_classifier/constants.py +++ b/src/tool_classifier/constants.py @@ -107,3 +107,29 @@ DENSE_SCORE_GAP_THRESHOLD = 0.05 """Cosine score gap (top - second) for high-confidence classification. Ensures the top result is significantly better than the runner-up.""" + + +# ============================================================================ +# Agentic Loop — Continuation Threshold +# ============================================================================ + +CONTINUATION_TURN = 3 +"""1-based turn count (after increment) at which the loop asks the user whether +to continue collecting parameters or fall back to the RAG workflow. +Only triggers when required params are still missing at exactly this turn. + +The turn counter is incremented on every run_turn() call, including the +initial call that generates the bot's opening question (before the user +speaks). With CONTINUATION_TURN=3 the conversation looks like: + + run_turn #1 (turn 0→1): initial question — "Which country and date?" + run_turn #2 (turn 1→2): user gives partial answer — bot asks follow-up + run_turn #3 (turn 2→3): user doesn't answer properly → CONTINUATION CHECK +""" + +CONTINUATION_QUESTION = ( + "I still need a bit more information, but we've been at this for a while. " + "Would you like to keep going and answer a few more questions, " + "or would you prefer to stop and get a general answer instead? (yes / no)" +) +"""Yes/no question shown to the user when the continuation threshold is reached.""" diff --git a/src/tool_classifier/context_analyzer.py b/src/tool_classifier/context_analyzer.py index 91b0a660..1b872487 100644 --- a/src/tool_classifier/context_analyzer.py +++ b/src/tool_classifier/context_analyzer.py @@ -10,8 +10,8 @@ from loguru import logger from pydantic import BaseModel, Field -from src.utils.cost_utils import get_lm_usage_since -from src.tool_classifier.greeting_constants import get_greeting_response +from utils.cost_utils import get_lm_usage_since +from tool_classifier.greeting_constants import get_greeting_response class ContextAnalysisResult(BaseModel): diff --git a/src/tool_classifier/enums.py b/src/tool_classifier/enums.py index ce6c7859..df8ced7f 100644 --- a/src/tool_classifier/enums.py +++ b/src/tool_classifier/enums.py @@ -37,3 +37,23 @@ class WorkflowType(Enum): WorkflowType.RAG: "RAG Workflow", WorkflowType.OOD: "Out-of-Domain Workflow", } + + +class AgenticLoopStatus(str, Enum): + """ + Status values returned by the agentic loop after each turn. + + - COMPLETED: All required parameters have been collected. + - NEEDS_INPUT: One or more required parameters are still missing; + a clarifying question is available for the user. + - MAX_TURNS_REACHED: The turn limit was hit before collection completed; + the caller should fall back gracefully. + - AWAITING_CONTINUATION_DECISION: The continuation threshold has been reached + with params still missing; a yes/no question is returned asking whether to + keep collecting or fall back to the RAG workflow. + """ + + COMPLETED = "completed" + NEEDS_INPUT = "needs_input" + MAX_TURNS_REACHED = "max_turns_reached" + AWAITING_CONTINUATION_DECISION = "awaiting_continuation_decision" diff --git a/src/tool_classifier/models.py b/src/tool_classifier/models.py index 9929473b..2d3d349b 100644 --- a/src/tool_classifier/models.py +++ b/src/tool_classifier/models.py @@ -1,9 +1,11 @@ """Data models for tool classifier system.""" +from dataclasses import dataclass from typing import Any, Dict, Optional + from pydantic import BaseModel, Field -from tool_classifier.enums import WorkflowType +from tool_classifier.enums import AgenticLoopStatus, WorkflowType class ClassificationResult(BaseModel): @@ -79,3 +81,23 @@ class ContextWorkflowMetadata(BaseModel): can_answer_from_history: bool = Field( default=False, description="Whether conversation history can answer this" ) + + +@dataclass +class AgenticLoopResult: + """ + Result returned by AgenticLoop.run_turn() after processing one conversation turn. + + Attributes: + status: Outcome of this turn — completed, needs_input, max_turns_reached, + or awaiting_continuation_decision. + collected_params: All parameters collected so far (prior turns + this turn merged). + clarifying_question: Natural-language question to show the user when status is + NEEDS_INPUT or AWAITING_CONTINUATION_DECISION. Empty string for other statuses. + turn_count: Updated turn counter (input turn_count + 1). + """ + + status: AgenticLoopStatus + collected_params: Dict[str, Any] + clarifying_question: str + turn_count: int diff --git a/tests/test_agentic_loop.py b/tests/test_agentic_loop.py new file mode 100644 index 00000000..80226e53 --- /dev/null +++ b/tests/test_agentic_loop.py @@ -0,0 +1,781 @@ +"""Unit tests for the AgenticLoop module.""" + +from typing import Any, Dict, List +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from tool_classifier.agentic_loop import AgenticLoop +from tool_classifier.enums import AgenticLoopStatus +from tool_classifier.param_extractor import ParamExtractionResult + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +_CHAT_ID = "test-chat-1" + +_SCHEMA_TWO_REQUIRED: List[Dict[str, Any]] = [ + { + "name": "countryIsoCode", + "type": "string", + "required": True, + "description": "Country", + }, + { + "name": "validFrom", + "type": "date", + "required": True, + "description": "Start date", + }, +] + +_SCHEMA_ONE_OPTIONAL: List[Dict[str, Any]] = [ + { + "name": "limit", + "type": "integer", + "required": False, + "description": "Max results", + }, +] + +_SCHEMA_EMPTY: List[Dict[str, Any]] = [] + +_HISTORY: List[Dict[str, Any]] = [ + {"authorRole": "user", "message": "Get me public holidays"}, + {"authorRole": "bot", "message": "Which country?"}, +] + + +def _make_session_store_mock() -> AsyncMock: + """Return an AsyncMock standing in for APIToolSessionStore.""" + mock = AsyncMock() + mock.update = AsyncMock(return_value=None) + return mock + + +def _make_extractor_mock(result: ParamExtractionResult) -> MagicMock: + """Return a MagicMock whose __call__() returns the given ParamExtractionResult.""" + mock = MagicMock(return_value=result) + return mock + + +def _make_loop( + extractor_mock: MagicMock, + session_store_mock: AsyncMock | None = None, +) -> AgenticLoop: + """Convenience factory that wires up AgenticLoop with mocked dependencies.""" + return AgenticLoop( + session_store=session_store_mock or _make_session_store_mock(), + param_extractor=extractor_mock, + ) + + +def _extraction( + extracted: Dict[str, Any], + missing: List[str], + question: str, +) -> ParamExtractionResult: + return ParamExtractionResult( + extracted_params=extracted, + missing_required=missing, + clarifying_question=question, + ) + + +# --------------------------------------------------------------------------- +# Turn limit guard +# --------------------------------------------------------------------------- + + +class TestMaxTurnsReached: + @pytest.mark.asyncio + async def test_max_turns_reached_when_turn_count_equals_max(self) -> None: + extractor_mock = _make_extractor_mock( + _extraction({}, ["countryIsoCode", "validFrom"], "Which country?") + ) + loop = _make_loop(extractor_mock) + + result = await loop.run_turn( + chat_id=_CHAT_ID, + user_message="hello", + conversation_history=_HISTORY, + params_schema=_SCHEMA_TWO_REQUIRED, + collected_params={}, + turn_count=5, + max_turns=5, + ) + + assert result.status == AgenticLoopStatus.MAX_TURNS_REACHED + extractor_mock.assert_not_called() + + @pytest.mark.asyncio + async def test_max_turns_reached_when_turn_count_exceeds_max(self) -> None: + extractor_mock = _make_extractor_mock( + _extraction({}, ["countryIsoCode"], "Which country?") + ) + loop = _make_loop(extractor_mock) + + result = await loop.run_turn( + chat_id=_CHAT_ID, + user_message="hello", + conversation_history=[], + params_schema=_SCHEMA_TWO_REQUIRED, + collected_params={"validFrom": "2026-01-01"}, + turn_count=10, + max_turns=5, + ) + + assert result.status == AgenticLoopStatus.MAX_TURNS_REACHED + assert result.collected_params == {"validFrom": "2026-01-01"} + extractor_mock.assert_not_called() + + @pytest.mark.asyncio + async def test_turn_count_incremented_on_max_turns(self) -> None: + loop = _make_loop(_make_extractor_mock(_extraction({}, [], "none"))) + + result = await loop.run_turn( + chat_id=_CHAT_ID, + user_message="hi", + conversation_history=[], + params_schema=_SCHEMA_TWO_REQUIRED, + collected_params={}, + turn_count=5, + max_turns=5, + ) + + assert result.turn_count == 6 + + +# --------------------------------------------------------------------------- +# COMPLETED status +# --------------------------------------------------------------------------- + + +class TestCompleted: + @pytest.mark.asyncio + async def test_completed_when_extractor_finds_last_param(self) -> None: + extractor_mock = _make_extractor_mock( + _extraction({"validFrom": "2026-01-01"}, [], "none") + ) + loop = _make_loop(extractor_mock) + + result = await loop.run_turn( + chat_id=_CHAT_ID, + user_message="January 2026", + conversation_history=_HISTORY, + params_schema=_SCHEMA_TWO_REQUIRED, + collected_params={"countryIsoCode": "EE"}, + turn_count=2, + max_turns=5, + ) + + assert result.status == AgenticLoopStatus.COMPLETED + assert result.collected_params == { + "countryIsoCode": "EE", + "validFrom": "2026-01-01", + } + assert result.clarifying_question == "" + + @pytest.mark.asyncio + async def test_completed_when_no_required_params_in_schema(self) -> None: + extractor_mock = _make_extractor_mock(_extraction({}, [], "none")) + loop = _make_loop(extractor_mock) + + result = await loop.run_turn( + chat_id=_CHAT_ID, + user_message="list all", + conversation_history=[], + params_schema=_SCHEMA_ONE_OPTIONAL, + collected_params={}, + turn_count=0, + max_turns=5, + ) + + assert result.status == AgenticLoopStatus.COMPLETED + + @pytest.mark.asyncio + async def test_completed_with_empty_schema(self) -> None: + extractor_mock = _make_extractor_mock(_extraction({}, [], "none")) + loop = _make_loop(extractor_mock) + + result = await loop.run_turn( + chat_id=_CHAT_ID, + user_message="go", + conversation_history=[], + params_schema=_SCHEMA_EMPTY, + collected_params={}, + turn_count=0, + ) + + assert result.status == AgenticLoopStatus.COMPLETED + + +# --------------------------------------------------------------------------- +# NEEDS_INPUT status +# --------------------------------------------------------------------------- + + +class TestNeedsInput: + @pytest.mark.asyncio + async def test_needs_input_when_params_still_missing(self) -> None: + extractor_mock = _make_extractor_mock( + _extraction( + {"countryIsoCode": "EE"}, + ["validFrom"], + "From which date?", + ) + ) + loop = _make_loop(extractor_mock) + + result = await loop.run_turn( + chat_id=_CHAT_ID, + user_message="Estonia", + conversation_history=_HISTORY, + params_schema=_SCHEMA_TWO_REQUIRED, + collected_params={}, + turn_count=0, + max_turns=5, + ) + + assert result.status == AgenticLoopStatus.NEEDS_INPUT + assert result.clarifying_question == "From which date?" + + @pytest.mark.asyncio + async def test_needs_input_when_extractor_finds_nothing(self) -> None: + extractor_mock = _make_extractor_mock( + _extraction({}, ["countryIsoCode", "validFrom"], "Which country and date?") + ) + loop = _make_loop(extractor_mock) + + result = await loop.run_turn( + chat_id=_CHAT_ID, + user_message="I want holidays", + conversation_history=[], + params_schema=_SCHEMA_TWO_REQUIRED, + collected_params={}, + turn_count=0, + ) + + assert result.status == AgenticLoopStatus.NEEDS_INPUT + assert result.clarifying_question == "Which country and date?" + + +# --------------------------------------------------------------------------- +# Param merging +# --------------------------------------------------------------------------- + + +class TestParamMerging: + @pytest.mark.asyncio + async def test_new_params_merged_with_prior_params(self) -> None: + extractor_mock = _make_extractor_mock( + _extraction({"validFrom": "2026-01-01"}, [], "none") + ) + loop = _make_loop(extractor_mock) + + result = await loop.run_turn( + chat_id=_CHAT_ID, + user_message="January 2026", + conversation_history=[], + params_schema=_SCHEMA_TWO_REQUIRED, + collected_params={"countryIsoCode": "EE"}, + turn_count=1, + ) + + assert result.collected_params == { + "countryIsoCode": "EE", + "validFrom": "2026-01-01", + } + + @pytest.mark.asyncio + async def test_prior_params_not_overwritten_by_extractor(self) -> None: + # Extractor tries to update countryIsoCode, but prior value is authoritative + extractor_mock = _make_extractor_mock( + _extraction({"countryIsoCode": "LV"}, [], "none") + ) + loop = _make_loop(extractor_mock) + + result = await loop.run_turn( + chat_id=_CHAT_ID, + user_message="Latvia", + conversation_history=[], + params_schema=_SCHEMA_TWO_REQUIRED, + collected_params={"countryIsoCode": "EE", "validFrom": "2026-01-01"}, + turn_count=2, + ) + + # Prior "EE" must not be overwritten by newly "extracted" "LV" + assert result.collected_params["countryIsoCode"] == "EE" + assert result.status == AgenticLoopStatus.COMPLETED + + @pytest.mark.asyncio + async def test_empty_extraction_preserves_prior_params(self) -> None: + extractor_mock = _make_extractor_mock( + _extraction({}, ["validFrom"], "From which date?") + ) + loop = _make_loop(extractor_mock) + + result = await loop.run_turn( + chat_id=_CHAT_ID, + user_message="not sure", + conversation_history=[], + params_schema=_SCHEMA_TWO_REQUIRED, + collected_params={"countryIsoCode": "EE"}, + turn_count=0, + ) + + assert result.collected_params == {"countryIsoCode": "EE"} + + +# --------------------------------------------------------------------------- +# Turn count +# --------------------------------------------------------------------------- + + +class TestTurnCount: + @pytest.mark.asyncio + async def test_turn_count_incremented_on_needs_input(self) -> None: + extractor_mock = _make_extractor_mock( + _extraction({}, ["countryIsoCode"], "Which country?") + ) + loop = _make_loop(extractor_mock) + + result = await loop.run_turn( + chat_id=_CHAT_ID, + user_message="hello", + conversation_history=[], + params_schema=_SCHEMA_TWO_REQUIRED, + collected_params={}, + turn_count=3, + ) + + assert result.turn_count == 4 + + @pytest.mark.asyncio + async def test_turn_count_incremented_on_completed(self) -> None: + extractor_mock = _make_extractor_mock( + _extraction({"countryIsoCode": "EE", "validFrom": "2026-01-01"}, [], "none") + ) + loop = _make_loop(extractor_mock) + + result = await loop.run_turn( + chat_id=_CHAT_ID, + user_message="Estonia, January 2026", + conversation_history=[], + params_schema=_SCHEMA_TWO_REQUIRED, + collected_params={}, + turn_count=0, + ) + + assert result.turn_count == 1 + + +# --------------------------------------------------------------------------- +# Session persistence +# --------------------------------------------------------------------------- + + +class TestSessionPersistence: + @pytest.mark.asyncio + async def test_session_saved_on_needs_input(self) -> None: + extractor_mock = _make_extractor_mock( + _extraction({"countryIsoCode": "EE"}, ["validFrom"], "From which date?") + ) + store_mock = _make_session_store_mock() + loop = _make_loop(extractor_mock, store_mock) + + await loop.run_turn( + chat_id=_CHAT_ID, + user_message="Estonia", + conversation_history=[], + params_schema=_SCHEMA_TWO_REQUIRED, + collected_params={}, + turn_count=0, + ) + + store_mock.update.assert_awaited_once_with( + _CHAT_ID, + collected_params={"countryIsoCode": "EE"}, + turn_count=1, + awaiting_continuation=False, + ) + + @pytest.mark.asyncio + async def test_session_saved_on_completed(self) -> None: + extractor_mock = _make_extractor_mock( + _extraction({"validFrom": "2026-01-01"}, [], "none") + ) + store_mock = _make_session_store_mock() + loop = _make_loop(extractor_mock, store_mock) + + await loop.run_turn( + chat_id=_CHAT_ID, + user_message="January 2026", + conversation_history=[], + params_schema=_SCHEMA_TWO_REQUIRED, + collected_params={"countryIsoCode": "EE"}, + turn_count=2, + ) + + store_mock.update.assert_awaited_once_with( + _CHAT_ID, + collected_params={"countryIsoCode": "EE", "validFrom": "2026-01-01"}, + turn_count=3, + awaiting_continuation=False, + ) + + @pytest.mark.asyncio + async def test_session_not_saved_on_max_turns(self) -> None: + extractor_mock = _make_extractor_mock(_extraction({}, [], "none")) + store_mock = _make_session_store_mock() + loop = _make_loop(extractor_mock, store_mock) + + await loop.run_turn( + chat_id=_CHAT_ID, + user_message="hi", + conversation_history=[], + params_schema=_SCHEMA_TWO_REQUIRED, + collected_params={}, + turn_count=5, + max_turns=5, + ) + + store_mock.update.assert_not_awaited() + + @pytest.mark.asyncio + async def test_session_not_saved_on_extractor_error(self) -> None: + extractor_mock = MagicMock() + extractor_mock.side_effect = RuntimeError("LLM timeout") + store_mock = _make_session_store_mock() + loop = _make_loop(extractor_mock, store_mock) + + await loop.run_turn( + chat_id=_CHAT_ID, + user_message="Estonia", + conversation_history=[], + params_schema=_SCHEMA_TWO_REQUIRED, + collected_params={}, + turn_count=1, + ) + + store_mock.update.assert_not_awaited() + + @pytest.mark.asyncio + async def test_session_save_failure_does_not_raise(self) -> None: + extractor_mock = _make_extractor_mock( + _extraction({}, ["countryIsoCode"], "Which country?") + ) + store_mock = _make_session_store_mock() + store_mock.update.side_effect = RuntimeError("Redis unavailable") + loop = _make_loop(extractor_mock, store_mock) + + # Should not raise even if Redis save fails + result = await loop.run_turn( + chat_id=_CHAT_ID, + user_message="hi", + conversation_history=[], + params_schema=_SCHEMA_TWO_REQUIRED, + collected_params={}, + turn_count=0, + ) + + assert result.status == AgenticLoopStatus.NEEDS_INPUT + + +# --------------------------------------------------------------------------- +# Error handling +# --------------------------------------------------------------------------- + + +class TestErrorHandling: + @pytest.mark.asyncio + async def test_extractor_exception_returns_needs_input(self) -> None: + extractor_mock = MagicMock() + extractor_mock.side_effect = RuntimeError("LLM timeout") + loop = _make_loop(extractor_mock) + + result = await loop.run_turn( + chat_id=_CHAT_ID, + user_message="Estonia", + conversation_history=[], + params_schema=_SCHEMA_TWO_REQUIRED, + collected_params={"validFrom": "2026-01-01"}, + turn_count=1, + ) + + assert result.status == AgenticLoopStatus.NEEDS_INPUT + assert result.clarifying_question == "" + # Prior collected_params preserved on error + assert result.collected_params == {"validFrom": "2026-01-01"} + assert result.turn_count == 2 + + @pytest.mark.asyncio + async def test_extractor_called_with_correct_arguments(self) -> None: + extractor_mock = _make_extractor_mock( + _extraction({"countryIsoCode": "EE"}, ["validFrom"], "From which date?") + ) + + with patch("tool_classifier.agentic_loop.asyncio.to_thread") as mock_to_thread: + # Make to_thread call the function synchronously so we can inspect args + async def fake_to_thread(fn: Any, *args: Any, **kwargs: Any) -> Any: + return fn(*args, **kwargs) + + mock_to_thread.side_effect = fake_to_thread + + loop = _make_loop(extractor_mock) + await loop.run_turn( + chat_id=_CHAT_ID, + user_message="Estonia", + conversation_history=_HISTORY, + params_schema=_SCHEMA_TWO_REQUIRED, + collected_params={"validFrom": "2026-01-01"}, + turn_count=1, + ) + + extractor_mock.assert_called_once_with( + "Estonia", + _SCHEMA_TWO_REQUIRED, + _HISTORY, + {"validFrom": "2026-01-01"}, + ) + + +# --------------------------------------------------------------------------- +# Continuation decision (turn-3 yes/no prompt) +# --------------------------------------------------------------------------- + + +class TestContinuationDecision: + @pytest.mark.asyncio + async def test_continuation_question_asked_at_threshold(self) -> None: + """AWAITING_CONTINUATION_DECISION is returned on exactly continuation_turn=3.""" + extractor_mock = _make_extractor_mock( + _extraction({}, ["countryIsoCode", "validFrom"], "Which country and date?") + ) + loop = _make_loop(extractor_mock) + + result = await loop.run_turn( + chat_id=_CHAT_ID, + user_message="I don't know", + conversation_history=_HISTORY, + params_schema=_SCHEMA_TWO_REQUIRED, + collected_params={}, + turn_count=2, # updated_turn_count == 3 == continuation_turn + max_turns=5, + ) + + assert result.status == AgenticLoopStatus.AWAITING_CONTINUATION_DECISION + assert result.clarifying_question != "" + assert result.turn_count == 3 + + @pytest.mark.asyncio + async def test_continuation_not_asked_before_threshold(self) -> None: + """Normal NEEDS_INPUT before the continuation threshold.""" + extractor_mock = _make_extractor_mock( + _extraction({}, ["countryIsoCode", "validFrom"], "Which country and date?") + ) + loop = _make_loop(extractor_mock) + + result = await loop.run_turn( + chat_id=_CHAT_ID, + user_message="hmm", + conversation_history=[], + params_schema=_SCHEMA_TWO_REQUIRED, + collected_params={}, + turn_count=0, # updated_turn_count == 1, below threshold + ) + + assert result.status == AgenticLoopStatus.NEEDS_INPUT + + @pytest.mark.asyncio + async def test_continuation_not_asked_after_threshold(self) -> None: + """Normal NEEDS_INPUT after the continuation threshold (user already continued).""" + extractor_mock = _make_extractor_mock( + _extraction({}, ["countryIsoCode", "validFrom"], "Which country and date?") + ) + loop = _make_loop(extractor_mock) + + result = await loop.run_turn( + chat_id=_CHAT_ID, + user_message="still not sure", + conversation_history=[], + params_schema=_SCHEMA_TWO_REQUIRED, + collected_params={}, + turn_count=3, # updated_turn_count == 4, past threshold + ) + + assert result.status == AgenticLoopStatus.NEEDS_INPUT + + @pytest.mark.asyncio + async def test_user_yes_resets_flag_and_continues(self) -> None: + """When awaiting_continuation=True and user says 'yes', loop continues.""" + extractor_mock = _make_extractor_mock( + _extraction({}, ["countryIsoCode", "validFrom"], "Which country and date?") + ) + loop = _make_loop(extractor_mock) + + result = await loop.run_turn( + chat_id=_CHAT_ID, + user_message="yes", + conversation_history=[], + params_schema=_SCHEMA_TWO_REQUIRED, + collected_params={}, + turn_count=3, + max_turns=5, + awaiting_continuation=True, + ) + + # Should continue normally — NEEDS_INPUT (params still missing after "yes") + assert result.status == AgenticLoopStatus.NEEDS_INPUT + + @pytest.mark.asyncio + async def test_user_estonian_yes_continues(self) -> None: + """Estonian 'jah' is recognised as an affirmative response.""" + extractor_mock = _make_extractor_mock( + _extraction({}, ["countryIsoCode", "validFrom"], "Mis riik?") + ) + loop = _make_loop(extractor_mock) + + result = await loop.run_turn( + chat_id=_CHAT_ID, + user_message="jah", + conversation_history=[], + params_schema=_SCHEMA_TWO_REQUIRED, + collected_params={}, + turn_count=3, + max_turns=5, + awaiting_continuation=True, + ) + + assert result.status == AgenticLoopStatus.NEEDS_INPUT + + @pytest.mark.asyncio + async def test_user_no_returns_max_turns_reached(self) -> None: + """When awaiting_continuation=True and user says 'no', RAG fallback is triggered.""" + extractor_mock = _make_extractor_mock( + _extraction({}, ["countryIsoCode"], "Which country?") + ) + loop = _make_loop(extractor_mock) + + result = await loop.run_turn( + chat_id=_CHAT_ID, + user_message="no", + conversation_history=[], + params_schema=_SCHEMA_TWO_REQUIRED, + collected_params={}, + turn_count=3, + max_turns=5, + awaiting_continuation=True, + ) + + assert result.status == AgenticLoopStatus.MAX_TURNS_REACHED + assert result.clarifying_question == "" + + @pytest.mark.asyncio + async def test_user_estonian_no_exits(self) -> None: + """Estonian 'ei' triggers RAG fallback.""" + extractor_mock = _make_extractor_mock( + _extraction({}, ["countryIsoCode"], "Mis riik?") + ) + loop = _make_loop(extractor_mock) + + result = await loop.run_turn( + chat_id=_CHAT_ID, + user_message="ei", + conversation_history=[], + params_schema=_SCHEMA_TWO_REQUIRED, + collected_params={}, + turn_count=3, + awaiting_continuation=True, + ) + + assert result.status == AgenticLoopStatus.MAX_TURNS_REACHED + + @pytest.mark.asyncio + async def test_ambiguous_response_exits(self) -> None: + """An ambiguous response while awaiting continuation defaults to exit.""" + extractor_mock = _make_extractor_mock( + _extraction({}, ["countryIsoCode"], "Which country?") + ) + loop = _make_loop(extractor_mock) + + result = await loop.run_turn( + chat_id=_CHAT_ID, + user_message="I'm not sure what to do", + conversation_history=[], + params_schema=_SCHEMA_TWO_REQUIRED, + collected_params={}, + turn_count=3, + awaiting_continuation=True, + ) + + assert result.status == AgenticLoopStatus.MAX_TURNS_REACHED + + @pytest.mark.asyncio + async def test_exit_preserves_collected_params(self) -> None: + """Collected params are returned unchanged when the user chooses to exit.""" + extractor_mock = _make_extractor_mock( + _extraction({}, ["validFrom"], "Which date?") + ) + loop = _make_loop(extractor_mock) + + result = await loop.run_turn( + chat_id=_CHAT_ID, + user_message="no", + conversation_history=[], + params_schema=_SCHEMA_TWO_REQUIRED, + collected_params={"countryIsoCode": "EE"}, + turn_count=3, + awaiting_continuation=True, + ) + + assert result.status == AgenticLoopStatus.MAX_TURNS_REACHED + assert result.collected_params == {"countryIsoCode": "EE"} + + @pytest.mark.asyncio + async def test_session_saved_with_awaiting_continuation_true(self) -> None: + """Session is persisted with awaiting_continuation=True at the threshold.""" + extractor_mock = _make_extractor_mock( + _extraction({}, ["countryIsoCode", "validFrom"], "Which country and date?") + ) + store_mock = _make_session_store_mock() + loop = _make_loop(extractor_mock, store_mock) + + await loop.run_turn( + chat_id=_CHAT_ID, + user_message="no idea", + conversation_history=[], + params_schema=_SCHEMA_TWO_REQUIRED, + collected_params={}, + turn_count=2, + ) + + store_mock.update.assert_awaited_once_with( + _CHAT_ID, + collected_params={}, + turn_count=3, + awaiting_continuation=True, + ) + + @pytest.mark.asyncio + async def test_session_not_saved_on_user_exit(self) -> None: + """Session is NOT saved when the user chooses to exit (caller deletes it).""" + extractor_mock = _make_extractor_mock( + _extraction({}, ["countryIsoCode"], "Which country?") + ) + store_mock = _make_session_store_mock() + loop = _make_loop(extractor_mock, store_mock) + + await loop.run_turn( + chat_id=_CHAT_ID, + user_message="no", + conversation_history=[], + params_schema=_SCHEMA_TWO_REQUIRED, + collected_params={}, + turn_count=3, + awaiting_continuation=True, + ) + + store_mock.update.assert_not_awaited() From 622c969ff56ed1a20906764389a0e6807fa77de2 Mon Sep 17 00:00:00 2001 From: nuwangeek Date: Fri, 17 Apr 2026 13:12:05 +0530 Subject: [PATCH 2/6] fixed requested changes --- src/tool_classifier/agentic_loop.py | 9 ++++++--- src/tool_classifier/context_analyzer.py | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/tool_classifier/agentic_loop.py b/src/tool_classifier/agentic_loop.py index 08915b75..0359f37c 100644 --- a/src/tool_classifier/agentic_loop.py +++ b/src/tool_classifier/agentic_loop.py @@ -114,8 +114,11 @@ async def run_turn( The returned turn_count is always input turn_count + 1. Session state is saved automatically on COMPLETED, NEEDS_INPUT, and AWAITING_CONTINUATION_DECISION. - It is NOT saved on MAX_TURNS_REACHED or extraction errors — the caller - is expected to delete the session in those cases. + It is NOT saved on MAX_TURNS_REACHED. It is also generally not saved + on extraction errors, except when a continuation decision was consumed + and the cleared ``awaiting_continuation`` state must be persisted. The + caller is expected to delete the session on MAX_TURNS_REACHED and + extraction errors after handling the failure. Args: chat_id: Unique conversation identifier, used as the Redis session key. @@ -323,4 +326,4 @@ def _detect_continuation_response(self, user_message: str) -> bool: True if the user wants to continue, False otherwise. """ normalised = user_message.strip().lower() - return normalised in _YES_RESPONSES + return normalised in _YES_RESPONSES \ No newline at end of file diff --git a/src/tool_classifier/context_analyzer.py b/src/tool_classifier/context_analyzer.py index 1b872487..d20f2d09 100644 --- a/src/tool_classifier/context_analyzer.py +++ b/src/tool_classifier/context_analyzer.py @@ -10,7 +10,7 @@ from loguru import logger from pydantic import BaseModel, Field -from utils.cost_utils import get_lm_usage_since +from src.utils.cost_utils import get_lm_usage_since from tool_classifier.greeting_constants import get_greeting_response @@ -1050,4 +1050,4 @@ def get_fallback_greeting_response(self, language: str = "et") -> str: "et": "Tere! Kuidas ma saan sind aidata?", "en": "Hello! How can I help you?", } - return greetings.get(language, greetings["et"]) + return greetings.get(language, greetings["et"]) \ No newline at end of file From cf9723edd6ead00ace04797295a725ae82eb1a4b Mon Sep 17 00:00:00 2001 From: nuwangeek Date: Fri, 17 Apr 2026 13:12:27 +0530 Subject: [PATCH 3/6] fixed ruff format issues --- src/tool_classifier/agentic_loop.py | 2 +- src/tool_classifier/context_analyzer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/tool_classifier/agentic_loop.py b/src/tool_classifier/agentic_loop.py index 0359f37c..fd5ae1d6 100644 --- a/src/tool_classifier/agentic_loop.py +++ b/src/tool_classifier/agentic_loop.py @@ -326,4 +326,4 @@ def _detect_continuation_response(self, user_message: str) -> bool: True if the user wants to continue, False otherwise. """ normalised = user_message.strip().lower() - return normalised in _YES_RESPONSES \ No newline at end of file + return normalised in _YES_RESPONSES diff --git a/src/tool_classifier/context_analyzer.py b/src/tool_classifier/context_analyzer.py index d20f2d09..da4eba1d 100644 --- a/src/tool_classifier/context_analyzer.py +++ b/src/tool_classifier/context_analyzer.py @@ -1050,4 +1050,4 @@ def get_fallback_greeting_response(self, language: str = "et") -> str: "et": "Tere! Kuidas ma saan sind aidata?", "en": "Hello! How can I help you?", } - return greetings.get(language, greetings["et"]) \ No newline at end of file + return greetings.get(language, greetings["et"]) From 83c7500567999368de37bd70b16f4c87986523cd Mon Sep 17 00:00:00 2001 From: nuwangeek Date: Wed, 22 Apr 2026 10:01:09 +0530 Subject: [PATCH 4/6] complete API semantic searcher with ambiguous result handling and tool classifier routing --- .../rag-search/POST/api-tools/search.yml | 81 +++ constants.ini | 3 +- docs/API_TOOL_CALLING.md | 204 +++++- src/api_tool_indexer/constants.py | 22 +- src/api_tool_indexer/main_indexer.py | 20 +- src/llm_orchestration_service.py | 2 + src/llm_orchestration_service_api.py | 83 +++ .../context_manager.py | 24 +- src/llm_orchestrator_config/feature_flags.py | 12 +- src/models/request_models.py | 6 + src/tool_classifier/api_semantic_searcher.py | 652 +++++++++++++++++ src/tool_classifier/classifier.py | 187 ++++- src/tool_classifier/constants.py | 22 + src/tool_classifier/enums.py | 16 +- src/tool_classifier/workflows/__init__.py | 2 + .../workflows/api_tool_workflow.py | 136 ++++ tests/api_tool_eval/batch_index.py | 106 +++ tests/api_tool_eval/endpoints.json | 223 ++++++ tests/api_tool_eval/eval_search.py | 383 ++++++++++ tests/api_tool_eval/results.json | 668 ++++++++++++++++++ 20 files changed, 2804 insertions(+), 48 deletions(-) create mode 100644 DSL/Ruuter.public/rag-search/POST/api-tools/search.yml create mode 100644 src/tool_classifier/api_semantic_searcher.py create mode 100644 src/tool_classifier/workflows/api_tool_workflow.py create mode 100644 tests/api_tool_eval/batch_index.py create mode 100644 tests/api_tool_eval/endpoints.json create mode 100644 tests/api_tool_eval/eval_search.py create mode 100644 tests/api_tool_eval/results.json diff --git a/DSL/Ruuter.public/rag-search/POST/api-tools/search.yml b/DSL/Ruuter.public/rag-search/POST/api-tools/search.yml new file mode 100644 index 00000000..4c19cef2 --- /dev/null +++ b/DSL/Ruuter.public/rag-search/POST/api-tools/search.yml @@ -0,0 +1,81 @@ +declaration: + call: declare + version: 0.1 + description: "Search API tool endpoints using semantic (hybrid) search against api_tool_collection(test endpoint)" + method: post + accepts: json + returns: json + namespace: rag-search + allowlist: + body: + - field: query + type: string + description: "Natural-language user query to search API endpoints" + - field: top_k + type: integer + description: "Max number of results to return (default: 5)" + - field: environment + type: string + description: "Embedding environment (default: production)" + +extract_request_data: + assign: + query: ${incoming.body.query} + top_k: ${incoming.body.top_k || 5} + environment: ${incoming.body.environment || 'production'} + next: validate_query + +validate_query: + switch: + - condition: "${!query || query.trim() === ''}" + next: return_missing_query + next: execute_search + +return_missing_query: + assign: + error_data: + success: false + error: "MISSING_QUERY" + message: "'query' field is required and must be a non-empty string" + next: return_bad_request + +execute_search: + call: http.post + args: + url: "[#RAG_SEARCH_LLM_SERVICE]/api-tools/search" + body: + query: ${query} + top_k: ${top_k} + environment: ${environment} + result: search_result + on_error: handle_search_error + next: check_search_status + +check_search_status: + switch: + - condition: ${200 <= search_result.response.statusCodeValue && search_result.response.statusCodeValue < 300} + next: return_ok + next: handle_search_error + +handle_search_error: + assign: + error_data: + success: false + error: "SEARCH_FAILED" + message: "Semantic search failed. LLM service may be unavailable." + next: return_server_error + +return_ok: + status: 200 + return: ${search_result.response.body} + next: end + +return_bad_request: + status: 400 + return: ${error_data} + next: end + +return_server_error: + status: 500 + return: ${error_data} + next: end \ No newline at end of file diff --git a/constants.ini b/constants.ini index 3eb8ac0c..d4195a4f 100644 --- a/constants.ini +++ b/constants.ini @@ -12,4 +12,5 @@ DOMAIN=localhost DB_PASSWORD=dbadmin RAG_SEARCH_RUUTER_PUBLIC_INTERNAL_SERVICE=http://ruuter:8086/services SERVICE_DMAPPER_HBS=http://data-mapper:3000/hbs/rag-search -SERVICE_PROJECT_LAYER=services \ No newline at end of file +SERVICE_PROJECT_LAYER=services +RAG_SEARCH_LLM_SERVICE=http://llm-orchestration-service:8100 \ No newline at end of file diff --git a/docs/API_TOOL_CALLING.md b/docs/API_TOOL_CALLING.md index 6072a2df..7985eb4a 100644 --- a/docs/API_TOOL_CALLING.md +++ b/docs/API_TOOL_CALLING.md @@ -8,12 +8,12 @@ API Tool Calling enables the LLM module to discover and invoke external API endp in response to user queries. endpoints are registered, semantically indexed in Qdrant, and retrieved at query time using hybrid search. -The feature has two halves: -| Half | What it does | Status | +| component | What it does | Status | |---|---|---| -| **Indexing pipeline** | Takes an endpoint definition → enriches it with LLM context → stores hybrid vectors in Qdrant | Complete | -| **Tool classifier** | At query time, routes to the best matching endpoint via hybrid search | In progress | +| **Indexing pipeline** | Takes an endpoint definition → enriches it with LLM context → stores hybrid vectors in Qdrant | ✅ Complete | +| **Tool classifier** | At query time, routes to the best matching endpoint via hybrid search + LLM disambiguation | ✅ Complete | +| **Workflow executor** | Surfaces the matched endpoint; full agentic loop (param collection → API call) planned | 🔧 Partial (Task 10) | --- @@ -30,7 +30,11 @@ main_indexer.py (indexing pipeline) ↓ upsert api_tool_collection (Qdrant) ↑ query at runtime -ToolClassifier (src/tool_classifier/) +APISemanticSearcher (src/tool_classifier/api_semantic_searcher.py) + ↑ called by +ToolClassifier._try_api_tool_classification() + ↓ ClassificationResult(workflow=API_TOOL_CALLING) +APIToolWorkflowExecutor (src/tool_classifier/workflows/api_tool_workflow.py) ``` --- @@ -97,7 +101,7 @@ Defined in [DSL/CronManager/script/api_tool_indexer.sh](../DSL/CronManager/scrip **What it does (in order):** -1. Validates required env vars (`endpoint_id`, `name`, `description`) +1. Validates required env vars (`endpoint_id`, `name`, `description`, `url`) 2. Activates the pre-built Python venv at `/app/python_virtual_env` 3. Installs required packages via `uv pip install` (`httpx`, `pydantic`, `qdrant-client`, `loguru`) 4. Sets `PYTHONPATH` to include `/app/src` @@ -235,4 +239,192 @@ loop can execute the API call without an additional database round-trip. | `required` | bool | Whether the caller must supply this param | | `description` | str | Human-readable description | +--- + + +## Part 2 — Tool Classifier (Query-Time) + +### Overview + +At query time, `ToolClassifier` in [src/tool_classifier/classifier.py](../src/tool_classifier/classifier.py) the layer by layer execution happens + + +1. **Service search** → `intent_collections` (Qdrant) — existing Bürokratt services +2. **API Tool search** → `api_tool_collection` (Qdrant) — registered API tool endpoints + +API tool search (`_try_api_tool_classification`) is triggered when: +- `SERVICE_WORKFLOW_ENABLED=false` (service workflow disabled globally) +- Dense service search returns no results +- Service cosine score falls below `DENSE_MIN_THRESHOLD` + +It is **always** tried before falling back to Context/RAG. + +--- + +### Component: `APISemanticSearcher` + +Defined in [src/tool_classifier/api_semantic_searcher.py](../src/tool_classifier/api_semantic_searcher.py). + +Instantiated once in `ToolClassifier.__init__()` and reuses the shared Qdrant `httpx.AsyncClient`. + +**Constructor:** + +```python +APISemanticSearcher( + embedding_service=orchestration_service, # generates dense embeddings + qdrant_client=self._qdrant_client, # shared connection pool + disambiguator=None, # optional: inject for testing +) +``` + +**Key constants** (from `constants.py`): + +| Constant | Value | Purpose | +|---|---|---| +| `API_TOOL_COLLECTION` | `api_tool_collection` | Qdrant collection name | +| `API_TOOL_SEARCH_TOP_K` | `5` | Max hybrid results | +| `API_TOOL_MIN_THRESHOLD` | cosine threshold | Below this → no match | +| `API_TOOL_HIGH_CONFIDENCE_THRESHOLD` | cosine threshold | Above this → high confidence | +| `API_TOOL_SCORE_GAP_THRESHOLD` | gap threshold | Minimum lead over runner-up | + +--- + +### Search Flow: `APISemanticSearcher.search()` + +``` +User query + │ + ├─ precomputed_embedding provided? → reuse it (no extra API call) + └─ otherwise → generate dense embedding via embedding_service + │ + ▼ +Step 1: Dense search (api_tool_collection) + → Real cosine similarity scores per endpoint + │ + ├─ No results → return [] + ├─ top_cosine < API_TOOL_MIN_THRESHOLD → return [] + └─ continue + │ + ▼ +Step 2: Hybrid search (dense + sparse/BM25 + RRF) + → Best-ranked results by RRF fusion score + │ Falls back to dense results if hybrid returns nothing + │ + ▼ +Step 3: Annotate confidence for each hybrid result + │ + │ cosine lookup: dense_cosine_map[endpoint_id] + │ └─ fallback: point["cosine_score"] (sparse-driven result) + │ └─ skip if neither available + │ + │ effective_gap = this_cosine − best_other_cosine_in_dense + │ + ├─ i==0 AND cosine ≥ HIGH_THRESHOLD AND effective_gap ≥ GAP_THRESHOLD → "high" + ├─ cosine ≥ MIN_THRESHOLD → "medium" + └─ else → skip + │ + ▼ +Step 4: Resolve to exactly one result + ├─ high-confidence result exists → return immediately + ├─ single medium + large gap → return directly + └─ multiple medium OR small gap → LLM disambiguation + │ + └─ EndpointDisambiguatorModule (DSPy + asyncio.to_thread) + → picks winner or returns None + → None means no match → return [] +``` + +--- + +### Embedding Reuse + +When `ToolClassifier.classify()` already generated a dense embedding for the service +search, it passes it as `precomputed_embedding` to `_try_api_tool_classification`: + +```python +api_tool_result = await self._try_api_tool_classification( + query, request, precomputed_embedding=query_embedding +) +``` + +`APISemanticSearcher.search()` skips the embedding step entirely when this is provided, +saving one embedding API call per request. + +--- + +### LLM Disambiguation: `EndpointDisambiguatorModule` + +Used when multiple medium-confidence endpoints score similarly and no clear winner +can be determined from cosine scores alone. + +- DSPy `Predict` module with `EndpointDisambiguationSignature` +- Inputs: `user_query` + `candidates` (JSON list of `{endpoint_id, name, description, cosine_score}`) +- Output: `best_endpoint_id` — the winning `endpoint_id`, or `"none"` if no match +- Run via `asyncio.to_thread()` to avoid blocking the async event loop +- Understands Estonian, Russian, and English queries + +--- + +### Feature Flag + +API tool calling is gated by `FeatureFlags.API_TOOL_CALLING_WORKFLOW_ENABLED`. +When `false`, `_try_api_tool_classification` returns `None` immediately without +touching Qdrant. + +--- + +### Component: `APIToolWorkflowExecutor` + +Defined in [src/tool_classifier/workflows/api_tool_workflow.py](../src/tool_classifier/workflows/api_tool_workflow.py). + +Handles `WorkflowType.API_TOOL_CALLING` after `ToolClassifier.classify()` has set +`matched_endpoint` in the context dict. + +**Current behaviour (Task 4.1):** + +Reads `context["matched_endpoint"]` and returns a simple confirmation response: + +``` +**{name}**: {description} + +URL: {url} +``` + +**Planned (Task 10):** Full agentic loop — +session management → parameter collection dialog → external API call → response formatting. + +--- + +### End-to-End Flow (Query Time) + +``` +User: "What are the public holidays in Estonia?" + │ + ▼ +ToolClassifier.classify() + │ + ├─ Dense search (intent_collections) → low cosine → below threshold + │ + └─ _try_api_tool_classification() + │ + └─ APISemanticSearcher.search() + │ + ├─ Dense: get_national_holidays cosine=0.87 + ├─ Hybrid: get_national_holidays ranked #1 (RRF) + ├─ effective_gap large → confidence="high" + └─ return [APIToolSearchResult(name="get_national_holidays", ...)] + │ + └─ ClassificationResult( + workflow=API_TOOL_CALLING, + metadata={"matched_endpoint": {...}} + ) + │ + ▼ +ToolClassifier._execute_with_fallback_async() + │ + └─ APIToolWorkflowExecutor.execute_async(context={"matched_endpoint": {...}}) + │ + └─ OrchestrationResponse(content="**get_national_holidays**: ...") +``` + --- \ No newline at end of file diff --git a/src/api_tool_indexer/constants.py b/src/api_tool_indexer/constants.py index 81c00132..a50b954f 100644 --- a/src/api_tool_indexer/constants.py +++ b/src/api_tool_indexer/constants.py @@ -31,7 +31,9 @@ class ApiToolIndexerConstants: REQUEST_TIMEOUT = 60 # seconds # Context Enrichment Template - # Used to generate a rich semantic context for each endpoint before embedding + # Mirrors the service workflow (intent_data_enrichment/constants.py). + # Full template goes in chunk_prompt; document_prompt is left empty. + # The LLM summarises the chunk content into a rich semantic context. CONTEXT_TEMPLATE = """ {full_endpoint_info} @@ -44,12 +46,24 @@ class ApiToolIndexerConstants: Please generate a rich, detailed context that describes this API endpoint comprehensively for semantic search. -Include information about: +Keep the prose context general and country-agnostic. Include information about: - What the user wants to accomplish by calling this endpoint - Key terms and synonyms for this action - Related concepts and use cases - Common ways users might ask for this functionality in natural language -IMPORTANT: Generate the context in the SAME LANGUAGE as the endpoint description above. If the description is in Estonian, respond in Estonian. If in English, respond in English. If in Russian, respond in Russian. +Then, on a new line, add a section exactly as shown below with 6 to 8 realistic and diverse example questions a real user might ask when they need this endpoint. Cover different phrasings, synonyms, and indirect ways of asking — do not just repeat the description verbatim. -Answer only with the enriched context and nothing else.""" +IMPORTANT for example queries: This is a system built for Estonian government digital services (Bürokratt). Ground the examples in an Estonian context — use Estonian cities (Tallinn, Tartu, Pärnu, Narva), Estonian institutions, and Estonia-relevant scenarios. Only use non-Estonian locations if the endpoint is explicitly about comparing or fetching data for multiple countries. + +Example queries: +- +- +- +- +- +- + +IMPORTANT: Generate everything in the SAME LANGUAGE as the endpoint description above. If the description is in Estonian, respond in Estonian. If in English, respond in English. If in Russian, respond in Russian. + +Answer only with the enriched context and example queries — nothing else.""" diff --git a/src/api_tool_indexer/main_indexer.py b/src/api_tool_indexer/main_indexer.py index 0a19f513..fe25d765 100644 --- a/src/api_tool_indexer/main_indexer.py +++ b/src/api_tool_indexer/main_indexer.py @@ -126,13 +126,22 @@ async def _generate_context_for_endpoint( params_summary=params_summary, ) - # Re-use the internal HTTP call of LLMAPIClient - /generate-context endpoint + logger.debug( + "Generated context prompt for endpoint '{}': {} chars", + endpoint_data.endpoint_id, + len(context_prompt), + ) + + # context_type="api_tool" makes context_manager use API_TOOL_CONTEXT_PROMPT, + # which passes chunk_prompt through unmodified so CHUNK_CONTEXT_PROMPT cannot + # override the instructions in CONTEXT_TEMPLATE (e.g. example query generation). request_data = { "document_prompt": "", "chunk_prompt": context_prompt, "environment": api_client.environment, - "use_cache": True, + "use_cache": False, "connection_id": api_client.connection_id, + "context_type": "api_tool", } last_error = None @@ -153,6 +162,13 @@ async def _generate_context_for_endpoint( result = response.json() context = result.get("context", "").strip() + + logger.debug( + "context preview: {}{}", + context[:200].replace("\n", "\\n"), + "..." if len(context) > 200 else "", + ) + if not context: raise ValueError("Empty context returned from API") diff --git a/src/llm_orchestration_service.py b/src/llm_orchestration_service.py index b4ce0058..91629baa 100644 --- a/src/llm_orchestration_service.py +++ b/src/llm_orchestration_service.py @@ -417,6 +417,7 @@ async def process_orchestration_request( query=request.message, conversation_history=request.conversationHistory, language=detected_language, + request=request, ) time_metric["classifier.classify"] = time.time() - start_time @@ -698,6 +699,7 @@ async def stream_orchestration_response( query=request.message, conversation_history=request.conversationHistory, language=detected_language, + request=request, ) time_metric["classifier.classify"] = time.time() - start_time diff --git a/src/llm_orchestration_service_api.py b/src/llm_orchestration_service_api.py index ddd66a9a..e8dddb7e 100644 --- a/src/llm_orchestration_service_api.py +++ b/src/llm_orchestration_service_api.py @@ -257,6 +257,89 @@ async def health_check(request: Request) -> dict[str, str]: } +@app.post( + "/api-tools/search", + status_code=status.HTTP_200_OK, + summary="[TEST] Search API tool endpoints by natural-language query", + description=( + "Test-only endpoint for evaluating semantic retrieval accuracy against " + "api_tool_collection. Bypasses classifier and all other workflows. " + "Returns ranked endpoints with cosine scores and confidence levels." + ), +) +async def api_tools_search( + http_request: Request, + body: Dict[str, Any], +) -> Dict[str, Any]: + """Run hybrid semantic search against api_tool_collection. + + Use this endpoint from Postman to evaluate whether the correct API endpoint + is returned for a given natural-language query. + + Request body: + query (str): Natural-language user query. Required. + top_k (int): Max results to return. Default: 5. + environment (str): Embedding environment. Default: "production". + + Response fields per result: + endpoint_id: UUID of the matched endpoint + name: Endpoint function name + description: Human-readable description + method: HTTP method (GET / POST) + url: Actual API URL + params: List of parameter schemas + cosine_score: How similar the query is to this endpoint (0.0 - 1.0) + confidence: "high" / "medium" (see threshold constants) + """ + from tool_classifier.api_semantic_searcher import APISemanticSearcher + + query = body.get("query", "").strip() + if not query: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="'query' field is required and must be a non-empty string", + ) + + top_k = int(body.get("top_k", 5)) + environment = body.get("environment", "production") + + orchestration_service = getattr( + http_request.app.state, "orchestration_service", None + ) + if orchestration_service is None: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Orchestration service not initialized", + ) + + try: + searcher = APISemanticSearcher(embedding_service=orchestration_service) + results = await searcher.search( + query=query, + environment=environment, + top_k=top_k, + ) + await searcher.aclose() + + return { + "query": query, + "total_results": len(results), + "results": [r.to_dict() for r in results], + "interpretation": { + "high_confidence": "Endpoint can be used directly — query is a very clear match", + "medium_confidence": "Possible match — may need LLM disambiguation in production", + "no_results": "No endpoint matched above the minimum threshold (0.45)", + }, + } + + except Exception as e: + logger.error(f"API tools search failed: {e}", exc_info=True) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Search failed: {str(e)}", + ) + + @app.post( "/orchestrate", response_model=OrchestrationResponse, diff --git a/src/llm_orchestrator_config/context_manager.py b/src/llm_orchestrator_config/context_manager.py index a14447ec..83eb4e1d 100644 --- a/src/llm_orchestrator_config/context_manager.py +++ b/src/llm_orchestrator_config/context_manager.py @@ -25,6 +25,10 @@ class ContextGenerationManager: Please give a short succinct context to situate this chunk within the overall document for the purposes of improving search retrieval of the chunk. Answer only with the succinct context and nothing else.""" + # Used by the API Tool Calling indexer — passes the caller's prompt through + # unmodified so CHUNK_CONTEXT_PROMPT cannot override the caller's own instructions. + API_TOOL_CONTEXT_PROMPT = """{chunk_content}""" + def __init__(self, llm_manager: LLMManager) -> None: """Initialize context generation manager.""" self.llm_manager = llm_manager @@ -43,7 +47,7 @@ def generate_context_with_caching( # Prepare the full prompt using Anthropic's format full_prompt = self._prepare_anthropic_prompt( - request.document_prompt, request.chunk_prompt + request.document_prompt, request.chunk_prompt, request.context_type ) # For now, call LLM directly (caching structure ready for future) @@ -103,8 +107,22 @@ def _resolve_model_for_request( logger.error(f"Failed to resolve model for context generation: {e}") raise RuntimeError(f"Model resolution failed: {e}") from e - def _prepare_anthropic_prompt(self, document_prompt: str, chunk_prompt: str) -> str: - """Prepare prompt in Anthropic's exact format.""" + def _prepare_anthropic_prompt( + self, + document_prompt: str, + chunk_prompt: str, + context_type: str = "chunk", + ) -> str: + """Prepare the LLM prompt based on context_type. + + - 'api_tool': returns chunk_prompt as-is using API_TOOL_CONTEXT_PROMPT so the + caller's own instructions (including example-query generation) are not + overridden by CHUNK_CONTEXT_PROMPT's closing directive. + - 'chunk' (default): uses the Anthropic RAG format (document + chunk sections). + """ + if context_type == "api_tool": + return self.API_TOOL_CONTEXT_PROMPT.format(chunk_content=chunk_prompt) + # Format document section document_section = self.DOCUMENT_CONTEXT_PROMPT.format( doc_content=document_prompt diff --git a/src/llm_orchestrator_config/feature_flags.py b/src/llm_orchestrator_config/feature_flags.py index d0d3fff8..6e5fac5b 100644 --- a/src/llm_orchestrator_config/feature_flags.py +++ b/src/llm_orchestrator_config/feature_flags.py @@ -20,7 +20,8 @@ class FeatureFlags: Environment Variables: - TOOL_CLASSIFIER_ENABLED: Master switch for classifier (default: false) - SERVICE_WORKFLOW_ENABLED: Enable Layer 1 service workflow (default: true) - - CONTEXT_WORKFLOW_ENABLED: Enable Layer 2 context workflow (default: true) + - API_TOOL_CALLING_WORKFLOW_ENABLED: Enable Layer 2 API tool calling workflow (default: true) + - CONTEXT_WORKFLOW_ENABLED: Enable Layer 3 context workflow (default: true) """ # Master switch for tool classifier @@ -35,6 +36,9 @@ class FeatureFlags: SERVICE_WORKFLOW_ENABLED = ( os.getenv("SERVICE_WORKFLOW_ENABLED", "true").lower() == "true" ) + API_TOOL_CALLING_WORKFLOW_ENABLED = ( + os.getenv("API_TOOL_CALLING_WORKFLOW_ENABLED", "true").lower() == "true" + ) CONTEXT_WORKFLOW_ENABLED = ( os.getenv("CONTEXT_WORKFLOW_ENABLED", "true").lower() == "true" ) @@ -53,6 +57,9 @@ def log_configuration(cls): logger.info(f" TOOL_CLASSIFIER_ENABLED: {cls.TOOL_CLASSIFIER_ENABLED}") if cls.TOOL_CLASSIFIER_ENABLED: logger.info(f" SERVICE_WORKFLOW_ENABLED: {cls.SERVICE_WORKFLOW_ENABLED}") + logger.info( + f" API_TOOL_CALLING_WORKFLOW_ENABLED: {cls.API_TOOL_CALLING_WORKFLOW_ENABLED}" + ) logger.info(f" CONTEXT_WORKFLOW_ENABLED: {cls.CONTEXT_WORKFLOW_ENABLED}") logger.info(f" FALLBACK_TO_RAG_ON_ERROR: {cls.FALLBACK_TO_RAG_ON_ERROR}") else: @@ -64,7 +71,7 @@ def is_workflow_enabled(cls, workflow_name: str) -> bool: Check if a specific workflow is enabled. Args: - workflow_name: Name of workflow ("service", "context", "rag", "ood") + workflow_name: Name of workflow ("service", "api_tool_calling", "context", "rag", "ood") Returns: True if workflow is enabled and classifier is enabled @@ -74,6 +81,7 @@ def is_workflow_enabled(cls, workflow_name: str) -> bool: workflow_flags = { "service": cls.SERVICE_WORKFLOW_ENABLED, + "api_tool_calling": cls.API_TOOL_CALLING_WORKFLOW_ENABLED, "context": cls.CONTEXT_WORKFLOW_ENABLED, "rag": True, # Always enabled "ood": True, # Always enabled diff --git a/src/models/request_models.py b/src/models/request_models.py index c6c58ebc..381f61a2 100644 --- a/src/models/request_models.py +++ b/src/models/request_models.py @@ -221,6 +221,12 @@ class ContextGenerationRequest(BaseModel): temperature: float = Field( default=0.1, description="Temperature for response generation", ge=0.0, le=2.0 ) + context_type: Literal["chunk", "api_tool"] = Field( + default="chunk", + description="Controls which prompt template is used. 'chunk' uses the Anthropic " + "RAG template (short succinct context). 'api_tool' passes the prompt through " + "unmodified so the caller's own instructions are respected.", + ) class ContextGenerationResponse(BaseModel): diff --git a/src/tool_classifier/api_semantic_searcher.py b/src/tool_classifier/api_semantic_searcher.py new file mode 100644 index 00000000..9b197d40 --- /dev/null +++ b/src/tool_classifier/api_semantic_searcher.py @@ -0,0 +1,652 @@ +"""API Tool Semantic Searcher — hybrid search against api_tool_collection.""" + +import asyncio +import json +from typing import Any, Dict, List, Optional, Protocol, cast + +import dspy +import httpx +from loguru import logger + +from tool_classifier.constants import ( + API_TOOL_COLLECTION, + API_TOOL_HIGH_CONFIDENCE_THRESHOLD, + API_TOOL_MIN_THRESHOLD, + API_TOOL_SCORE_GAP_THRESHOLD, + API_TOOL_SEARCH_TOP_K, + QDRANT_HOST, + QDRANT_PORT, + QDRANT_TIMEOUT, +) +from tool_classifier.sparse_encoder import compute_sparse_vector + + +class EmbeddingServiceProtocol(Protocol): + """Protocol for any service that can generate text embeddings.""" + + def create_embeddings_for_indexer( + self, + texts: List[str], + environment: str = "production", + connection_id: Optional[str] = None, + batch_size: int = 10, + ) -> Dict[str, Any]: ... + + +class APIToolSearchResult: + """Result from API Tool semantic search.""" + + def __init__( + self, + endpoint_id: str, + name: str, + description: str, + method: str, + url: str, + params: List[Dict], + cosine_score: float, + rrf_score: float, + confidence: str, + ) -> None: + self.endpoint_id = endpoint_id + self.name = name + self.description = description + self.method = method + self.url = url + self.params = params + self.cosine_score = ( + cosine_score # Real dense cosine similarity (used for thresholds) + ) + self.rrf_score = rrf_score # Hybrid RRF fusion score (used for ranking) + self.confidence = confidence # "high", "medium", "none" + + def to_dict(self) -> Dict[str, Any]: + return { + "endpoint_id": self.endpoint_id, + "name": self.name, + "description": self.description, + "method": self.method, + "url": self.url, + "params": self.params, + "cosine_score": round(self.cosine_score, 4), + "rrf_score": round(self.rrf_score, 6), + "confidence": self.confidence, + } + + +class EndpointDisambiguationSignature(dspy.Signature): + """Determine which API endpoint best matches a user query, or none. + + Rules: + - Analyze the user query against the candidate endpoints carefully + - Return the endpoint_id of the best match if one clearly addresses the query + - Return exactly "none" if no endpoint clearly fits — do not guess + - Be conservative — only match when confident + - Understand Estonian, Russian, and English queries + """ + + user_query: str = dspy.InputField( + desc="User's question or request in Estonian, Russian, or English" + ) + candidates: str = dspy.InputField( + desc="JSON list of candidate endpoints: [{endpoint_id, name, description, cosine_score}]" + ) + + best_endpoint_id: str = dspy.OutputField( + desc='The endpoint_id of the best match, or exactly "none" if no endpoint clearly fits' + ) + + +class EndpointDisambiguatorModule(dspy.Module): + """DSPy Module for resolving ambiguous API endpoint candidates via LLM. + + Called when multiple endpoints score in the medium-confidence range and + a clear winner cannot be determined from cosine scores alone. + """ + + def __init__(self) -> None: + """Initialize with a direct DSPy predictor.""" + super().__init__() + self.predictor = dspy.Predict(EndpointDisambiguationSignature) + + def forward( + self, + user_query: str, + candidates: List[Dict[str, Any]], + ) -> Optional[str]: + """Pick the best matching endpoint_id from candidates, or return None. + + Args: + user_query: The user's natural language query. + candidates: List of candidate dicts with endpoint_id, name, + description, and cosine_score. + + Returns: + The winning endpoint_id string, or None if no endpoint clearly fits. + """ + candidates_payload = [ + { + "endpoint_id": c["endpoint_id"], + "name": c["name"], + "description": c["description"], + "cosine_score": round(c["cosine_score"], 4), + } + for c in candidates + ] + candidates_json = json.dumps(candidates_payload, ensure_ascii=False, indent=2) + + try: + result = self.predictor( + user_query=user_query, + candidates=candidates_json, + ) + winner = result.best_endpoint_id.strip() + if winner.lower() == "none": + return None + return winner + except Exception as e: + logger.error( + f"EndpointDisambiguatorModule: Disambiguation failed: {e}", + exc_info=True, + ) + return None + + +class APISemanticSearcher: + """Semantic searcher for API Tool endpoints stored in api_tool_collection. + + Usage: + searcher = APISemanticSearcher( + qdrant_client=shared_httpx_client, + embedding_service=orchestration_service, + ) + results = await searcher.search("What are national holidays in Estonia?") + """ + + def __init__( + self, + embedding_service: EmbeddingServiceProtocol, + qdrant_client: Optional[httpx.AsyncClient] = None, + disambiguator: Optional[EndpointDisambiguatorModule] = None, + ) -> None: + """Initialize the API semantic searcher. + + Args: + embedding_service: Service that generates dense embeddings. + qdrant_client: Optional shared httpx client. If None, creates its own. + disambiguator: Optional DSPy disambiguation module. If None, a default + instance is created. Inject a custom instance for testing. + """ + self.embedding_service = embedding_service + self._disambiguator = ( + disambiguator + if disambiguator is not None + else EndpointDisambiguatorModule() + ) + self._owns_client = qdrant_client is None + + if qdrant_client is not None: + self._qdrant_client = qdrant_client + else: + self._qdrant_client = httpx.AsyncClient( + base_url=f"http://{QDRANT_HOST}:{QDRANT_PORT}", + timeout=QDRANT_TIMEOUT, + limits=httpx.Limits( + max_connections=10, + max_keepalive_connections=5, + ), + ) + + async def aclose(self) -> None: + """Close the httpx client if we own it.""" + if self._owns_client: + await self._qdrant_client.aclose() + + async def search( + self, + query: str, + environment: str = "production", + connection_id: Optional[str] = None, + top_k: int = API_TOOL_SEARCH_TOP_K, + precomputed_embedding: Optional[List[float]] = None, + ) -> List[APIToolSearchResult]: + """Search api_tool_collection for the best matching API endpoints. + + Uses a two-step approach: + 1. Dense search → get real cosine similarity scores + 2. Hybrid search (dense + sparse + RRF) → get best-ranked matches + + Returns endpoints annotated with confidence level: + - "high": cosine >= API_TOOL_HIGH_CONFIDENCE_THRESHOLD AND score gap is large + - "medium": cosine >= API_TOOL_MIN_THRESHOLD but ambiguous + - "none": cosine < API_TOOL_MIN_THRESHOLD (no match) + + Args: + query: Natural language user query. + environment: Environment for embedding model resolution. + connection_id: Optional connection ID for embedding service. + top_k: Maximum number of results to return. + precomputed_embedding: Dense vector already computed upstream (e.g. by + the service classifier). When provided the embedding step is skipped + entirely, saving one embedding API call per request. + + Returns: + List containing exactly one APIToolSearchResult (the resolved best match), + or an empty list if no suitable API tool endpoint was found. + Never returns more than one result — ambiguous medium-confidence candidates + are resolved via LLM disambiguation before returning. + """ + # Step 1: Reuse caller's embedding if provided, otherwise generate a new one + if precomputed_embedding is not None: + logger.debug( + "APISemanticSearcher: reusing precomputed query embedding (no extra API call)" + ) + query_embedding = precomputed_embedding + else: + query_embedding = self._get_query_embedding( + query, environment, connection_id + ) + if query_embedding is None: + logger.error("APISemanticSearcher: Failed to generate query embedding") + return [] + + # Step 2: Dense search → real cosine scores for relevance check + dense_results = await self._dense_search(query_embedding, top_k=top_k) + if not dense_results: + logger.info("APISemanticSearcher: No results from dense search") + return [] + + top_cosine = dense_results[0]["cosine_score"] + second_cosine = ( + dense_results[1]["cosine_score"] if len(dense_results) > 1 else 0.0 + ) + cosine_gap = top_cosine - second_cosine + + logger.info(f"APISemanticSearcher: query={query!r}") + logger.info( + f"APISemanticSearcher: dense top={dense_results[0]['name']} " + f"(cosine={top_cosine:.4f}), gap={cosine_gap:.4f}" + ) + + # Below minimum threshold → no match + if top_cosine < API_TOOL_MIN_THRESHOLD: + logger.info( + f"APISemanticSearcher: cosine {top_cosine:.4f} < " + f"threshold {API_TOOL_MIN_THRESHOLD} — no API tool match" + ) + return [] + + # Step 3: Hybrid search → best-ranked results using dense + sparse + RRF + query_sparse = compute_sparse_vector(query) + hybrid_results = await self._hybrid_search( + query_embedding, query_sparse, top_k=top_k + ) + + # Fall back to dense results if hybrid returns nothing + if not hybrid_results: + hybrid_results = dense_results + + # Build a lookup from endpoint_id → real cosine score from dense results + dense_cosine_map = {r["endpoint_id"]: r["cosine_score"] for r in dense_results} + + # Step 4: Annotate each result with confidence level + results: List[APIToolSearchResult] = [] + for i, point in enumerate(hybrid_results): + endpoint_id = point.get("endpoint_id", "") + + # Prefer cosine from dense search; if this hybrid result was not in the + # dense top-N set, fall back to the cosine carried on the hybrid result + # itself. Skip entirely if no actual cosine score is available. + point_cosine = dense_cosine_map.get(endpoint_id) + if point_cosine is None: + point_cosine = point.get("cosine_score") + if point_cosine is None: + continue # Skip results that do not have an actual cosine score + point_rrf = point.get("rrf_score", 0.0) + + # Compute gap relative to this candidate: its cosine vs the best other + # dense cosine. This is correct even when hybrid re-ranks the top result. + next_best_cosine = next( + ( + r["cosine_score"] + for r in dense_results + if r["endpoint_id"] != endpoint_id + ), + 0.0, + ) + effective_gap = point_cosine - next_best_cosine + + if ( + point_cosine >= API_TOOL_HIGH_CONFIDENCE_THRESHOLD + and effective_gap >= API_TOOL_SCORE_GAP_THRESHOLD + and i == 0 + ): + confidence = "high" + elif point_cosine >= API_TOOL_MIN_THRESHOLD: + confidence = "medium" + else: + continue # Skip results below threshold + + results.append( + APIToolSearchResult( + endpoint_id=endpoint_id, + name=point.get("name", ""), + description=point.get("description", ""), + method=point.get("method", "GET"), + url=point.get("url", ""), + params=point.get("params", []), + cosine_score=point_cosine, + rrf_score=point_rrf, + confidence=confidence, + ) + ) + + # Step 5: Resolve to exactly one result + high_results = [r for r in results if r.confidence == "high"] + if high_results: + logger.info( + f"APISemanticSearcher: high-confidence match → {high_results[0].name!r} " + f"(cosine={high_results[0].cosine_score:.4f})" + ) + return [high_results[0]] + + medium_results = [r for r in results if r.confidence == "medium"] + if not medium_results: + logger.info( + "APISemanticSearcher: no results above threshold — no API tool match" + ) + return [] + + # Single medium result — only return directly if the gap is large enough + # (gap < SCORE_GAP_THRESHOLD means runner-up was close, LLM should validate) + if len(medium_results) == 1 and cosine_gap >= API_TOOL_SCORE_GAP_THRESHOLD: + logger.info( + f"APISemanticSearcher: single medium-confidence match (gap={cosine_gap:.4f}) → " + f"{medium_results[0].name!r} (cosine={medium_results[0].cosine_score:.4f})" + ) + return [medium_results[0]] + + # Multiple ambiguous candidates, OR single candidate with small gap — LLM validates + if len(medium_results) == 1: + logger.info( + f"APISemanticSearcher: single medium result but gap={cosine_gap:.4f} < " + f"{API_TOOL_SCORE_GAP_THRESHOLD} — sending to LLM for validation" + ) + winner_id = await self._disambiguate(query, medium_results) + if winner_id is None: + logger.info( + "APISemanticSearcher: disambiguator rejected all candidates — no API tool match" + ) + return [] + + winner = next((r for r in medium_results if r.endpoint_id == winner_id), None) + if winner is None: + logger.warning( + f"APISemanticSearcher: disambiguator returned unknown " + f"endpoint_id={winner_id!r} — no API tool match" + ) + return [] + + logger.info( + f"APISemanticSearcher: disambiguated winner → {winner.name!r} " + f"(cosine={winner.cosine_score:.4f})" + ) + return [winner] + + async def _disambiguate( + self, + query: str, + candidates: List[APIToolSearchResult], + ) -> Optional[str]: + """Invoke LLM disambiguation on ambiguous medium-confidence candidates. + + Args: + query: The original user query. + candidates: Medium-confidence APIToolSearchResult items to choose between. + + Returns: + The endpoint_id of the winner, or None if the LLM rejects all candidates. + """ + candidate_dicts = [ + { + "endpoint_id": r.endpoint_id, + "name": r.name, + "description": r.description, + "cosine_score": r.cosine_score, + } + for r in candidates + ] + logger.info( + f"APISemanticSearcher: disambiguating {len(candidates)} candidates " + f"for query: {query!r}" + ) + # Run the synchronous DSPy LLM call in a thread pool so it does not + # block the asyncio event loop while waiting for the LLM response. + # cast: asyncio.to_thread infers Prediction from DSPy; forward() returns Optional[str] + winner_id = cast( + Optional[str], + await asyncio.to_thread( + self._disambiguator, + user_query=query, + candidates=candidate_dicts, + ), + ) + if winner_id: + logger.info( + f"APISemanticSearcher: disambiguator picked endpoint_id={winner_id!r}" + ) + else: + logger.info("APISemanticSearcher: disambiguator rejected all candidates") + return winner_id + + def _get_query_embedding( + self, + query: str, + environment: str, + connection_id: Optional[str], + ) -> Optional[List[float]]: + """Generate dense embedding for the query.""" + try: + result = self.embedding_service.create_embeddings_for_indexer( + texts=[query], + environment=environment, + connection_id=connection_id, + batch_size=1, + ) + embeddings = result.get("embeddings", []) + if embeddings: + return embeddings[0] + logger.error("APISemanticSearcher: No embedding returned") + return None + except Exception as e: + logger.error(f"APISemanticSearcher: Embedding generation failed: {e}") + return None + + async def _dense_search( + self, + dense_vector: List[float], + top_k: int, + ) -> List[Dict[str, Any]]: + """Dense-only search on api_tool_collection for real cosine scores. + + Returns deduplicated results by endpoint_id, sorted by cosine score. + """ + try: + search_payload = { + "query": dense_vector, + "using": "dense", + "limit": top_k * 2, + "with_payload": True, + } + + response = await self._qdrant_client.post( + f"/collections/{API_TOOL_COLLECTION}/points/query", + json=search_payload, + ) + + if response.status_code != 200: + logger.error( + f"APISemanticSearcher: Dense search failed " + f"HTTP {response.status_code} — {response.text}" + ) + return [] + + points = response.json().get("result", {}).get("points", []) + if not points: + return [] + + # Deduplicate by endpoint_id, keep best cosine score + endpoint_results: Dict[str, Dict[str, Any]] = {} + for point in points: + payload = point.get("payload", {}) + score = float(point.get("score", 0)) + endpoint_id = payload.get("endpoint_id", "unknown") + + if endpoint_id not in endpoint_results or score > endpoint_results[ + endpoint_id + ].get("cosine_score", 0): + endpoint_results[endpoint_id] = { + "endpoint_id": endpoint_id, + "name": payload.get("name", ""), + "description": payload.get("description", ""), + "method": payload.get("method", "GET"), + "url": payload.get("url", ""), + "params": payload.get("params", []), + "cosine_score": score, + } + + return sorted( + endpoint_results.values(), + key=lambda x: x["cosine_score"], + reverse=True, + ) + + except httpx.TimeoutException: + logger.error( + f"APISemanticSearcher: Dense search timeout after {QDRANT_TIMEOUT}s" + ) + return [] + except Exception as e: + logger.error( + f"APISemanticSearcher: Dense search failed: {e}", exc_info=True + ) + return [] + + async def _hybrid_search( + self, + dense_vector: List[float], + sparse_vector: Any, + top_k: int, + ) -> List[Dict[str, Any]]: + """Hybrid search using dense + sparse + RRF fusion. + + Sends both vectors in a single Qdrant prefetch query. + Returns deduplicated results by endpoint_id, sorted by RRF score. + """ + try: + # Verify collection is non-empty before searching + collection_info = await self._qdrant_client.get( + f"/collections/{API_TOOL_COLLECTION}" + ) + if collection_info.status_code == 200: + points_count = ( + collection_info.json().get("result", {}).get("points_count", 0) + ) + if points_count == 0: + logger.info("APISemanticSearcher: api_tool_collection is empty") + return [] + else: + logger.warning( + f"APISemanticSearcher: Could not verify collection: " + f"HTTP {collection_info.status_code}" + ) + return [] + + # Build prefetch + RRF payload + search_payload: Dict[str, Any] = { + "prefetch": [ + { + "query": dense_vector, + "using": "dense", + "limit": top_k * 2, + }, + ], + "query": {"fusion": "rrf"}, + "limit": top_k, + "with_payload": True, + } + + # Add sparse prefetch only if non-empty + if not sparse_vector.is_empty(): + search_payload["prefetch"].append( + { + "query": sparse_vector.to_dict(), + "using": "sparse", + "limit": top_k * 2, + } + ) + + response = await self._qdrant_client.post( + f"/collections/{API_TOOL_COLLECTION}/points/query", + json=search_payload, + ) + + if response.status_code != 200: + logger.error( + f"APISemanticSearcher: Hybrid search failed " + f"HTTP {response.status_code} — {response.text}" + ) + return [] + + points = response.json().get("result", {}).get("points", []) + if not points: + return [] + + # Deduplicate by endpoint_id, keep best RRF score + endpoint_results: Dict[str, Dict[str, Any]] = {} + for point in points: + payload = point.get("payload", {}) + score = float(point.get("score", 0)) + endpoint_id = payload.get("endpoint_id", "unknown") + + if endpoint_id not in endpoint_results or score > endpoint_results[ + endpoint_id + ].get("rrf_score", 0): + endpoint_results[endpoint_id] = { + "endpoint_id": endpoint_id, + "name": payload.get("name", ""), + "description": payload.get("description", ""), + "method": payload.get("method", "GET"), + "url": payload.get("url", ""), + "params": payload.get("params", []), + "rrf_score": score, + # cosine_score patched in search() from dense results + } + + sorted_results = sorted( + endpoint_results.values(), + key=lambda x: x["rrf_score"], + reverse=True, + ) + + logger.info( + f"APISemanticSearcher: hybrid returned {len(sorted_results)} unique endpoints" + ) + for i, r in enumerate(sorted_results[:3]): + logger.debug( + f" Rank {i + 1}: {r['name']} " + f"(endpoint_id={r['endpoint_id']}, rrf={r['rrf_score']:.6f})" + ) + + return sorted_results + + except httpx.TimeoutException: + logger.error( + f"APISemanticSearcher: Hybrid search timeout after {QDRANT_TIMEOUT}s" + ) + return [] + except Exception as e: + logger.error( + f"APISemanticSearcher: Hybrid search failed: {e}", exc_info=True + ) + return [] diff --git a/src/tool_classifier/classifier.py b/src/tool_classifier/classifier.py index ff683f6f..06be08c7 100644 --- a/src/tool_classifier/classifier.py +++ b/src/tool_classifier/classifier.py @@ -27,12 +27,15 @@ DENSE_SCORE_GAP_THRESHOLD, ) from tool_classifier.sparse_encoder import compute_sparse_vector +from tool_classifier.api_semantic_searcher import APISemanticSearcher from tool_classifier.workflows import ( + APIToolWorkflowExecutor, ServiceWorkflowExecutor, ContextWorkflowExecutor, RAGWorkflowExecutor, OODWorkflowExecutor, ) +from llm_orchestrator_config.feature_flags import FeatureFlags class ToolClassifier: @@ -82,6 +85,9 @@ def __init__( ) # Initialize workflow executors + self.api_tool_workflow = APIToolWorkflowExecutor( + orchestration_service=orchestration_service, + ) self.service_workflow = ServiceWorkflowExecutor( llm_manager=llm_manager, orchestration_service=orchestration_service, @@ -95,6 +101,12 @@ def __init__( ) self.ood_workflow = OODWorkflowExecutor() + # API tool semantic searcher - reuses the shared Qdrant client + self.api_tool_searcher = APISemanticSearcher( + embedding_service=orchestration_service, + qdrant_client=self._qdrant_client, + ) + logger.info( "Tool classifier initialized with hybrid search classification " f"(Qdrant: {self._qdrant_base_url})" @@ -113,6 +125,7 @@ async def classify( query: str, conversation_history: List[ConversationItem], language: str, + request: Optional[OrchestrationRequest] = None, ) -> ClassificationResult: """ Classify a user query using a two-step search approach. @@ -121,14 +134,17 @@ async def classify( Step 2: Hybrid search (dense + sparse + RRF) → service identification Routing: - - cosine < DENSE_MIN_THRESHOLD → CONTEXT/RAG (skip SERVICE) + - cosine < DENSE_MIN_THRESHOLD AND no ATC match → CONTEXT/RAG - cosine ≥ HIGH_CONFIDENCE + large gap → SERVICE (no LLM needed) + - ATC match found (when SERVICE misses) → API_TOOL_CALLING - else → SERVICE with LLM confirmation Args: query: User's query string conversation_history: List of previous conversation messages language: Detected language code (e.g., 'en', 'et') + request: Original orchestration request (needed for ATC search + which requires environment and connection_id for embedding). Returns: ClassificationResult indicating which workflow to use @@ -136,6 +152,23 @@ async def classify( logger.info(f"Classifying query: {query[:100]}...") try: + if not FeatureFlags.SERVICE_WORKFLOW_ENABLED: + logger.info( + "SERVICE_WORKFLOW_ENABLED=false - skipping standard service search" + ) + api_tool_result = await self._try_api_tool_classification( + query, request + ) + if api_tool_result: + return api_tool_result + logger.info("No API tool match either — routing to CONTEXT/RAG") + return ClassificationResult( + workflow=WorkflowType.CONTEXT, + confidence=1.0, + metadata={"reason": "service_workflow_disabled"}, + reasoning="Service workflow disabled, no ATC match - fallback to Context/RAG", + ) + # Step 1: Generate dense embedding for query query_embedding = self._get_query_embedding(query) if query_embedding is None: @@ -156,7 +189,15 @@ async def classify( ) if not dense_results: - logger.info("No dense search results - routing to CONTEXT/RAG") + logger.info( + "No dense search results from intent_collections - trying API tools" + ) + api_tool_result = await self._try_api_tool_classification( + query, request, precomputed_embedding=query_embedding + ) + if api_tool_result: + return api_tool_result + logger.info("No API tool match either — routing to CONTEXT/RAG") return ClassificationResult( workflow=WorkflowType.CONTEXT, confidence=1.0, @@ -184,9 +225,15 @@ async def classify( # Decision: Is this a service query at all? if top_cosine < DENSE_MIN_THRESHOLD: logger.info( - f"Low relevance (cosine={top_cosine:.4f} < {DENSE_MIN_THRESHOLD}) " - f"- routing to CONTEXT/RAG, skipping SERVICE" + f"Low service relevance (cosine={top_cosine:.4f} < {DENSE_MIN_THRESHOLD}) " + f"— trying API tools before falling to CONTEXT/RAG" + ) + api_tool_result = await self._try_api_tool_classification( + query, request, precomputed_embedding=query_embedding ) + if api_tool_result: + return api_tool_result + logger.info("No API tool match — routing to CONTEXT/RAG") return ClassificationResult( workflow=WorkflowType.CONTEXT, confidence=1.0, @@ -615,12 +662,82 @@ def _get_workflow_executor(self, workflow_type: WorkflowType) -> Any: """Get workflow executor instance for given workflow type.""" workflow_map = { WorkflowType.SERVICE: self.service_workflow, + WorkflowType.API_TOOL_CALLING: self.api_tool_workflow, WorkflowType.CONTEXT: self.context_workflow, WorkflowType.RAG: self.rag_workflow, WorkflowType.OOD: self.ood_workflow, } return workflow_map[workflow_type] + def _is_workflow_enabled(self, workflow_type: WorkflowType) -> bool: + """Return True if the given workflow type is enabled via feature flags. + + RAG and OOD are always enabled (they are the safety net fallbacks). + """ + flag_map = { + WorkflowType.SERVICE: FeatureFlags.SERVICE_WORKFLOW_ENABLED, + WorkflowType.API_TOOL_CALLING: FeatureFlags.API_TOOL_CALLING_WORKFLOW_ENABLED, + WorkflowType.CONTEXT: FeatureFlags.CONTEXT_WORKFLOW_ENABLED, + WorkflowType.RAG: True, + WorkflowType.OOD: True, + } + return flag_map.get(workflow_type, True) + + async def _try_api_tool_classification( + self, + query: str, + request: Optional[OrchestrationRequest] = None, + precomputed_embedding: Optional[List[float]] = None, + ) -> Optional[ClassificationResult]: + """Search api_tool_collection and return a ClassificationResult if a match is found. + + Called when intent_collections search yields no usable service match. + + Args: + query: User's query string. + request: Orchestration request (provides environment + connection_id). + When None, defaults to production environment. + precomputed_embedding: Dense embedding vector already computed for this + query by the service search step. When provided, the ATC searcher + reuses it instead of making a second embedding API call. + + Returns: + ClassificationResult with API_TOOL_CALLING workflow if a match is found, + or None if no endpoint matched. + """ + if not FeatureFlags.API_TOOL_CALLING_WORKFLOW_ENABLED: + logger.info("API_TOOL_CALLING_WORKFLOW_ENABLED=false — skipping ATC search") + return None + + environment = request.environment if request else "production" + connection_id = request.connection_id if request else None + + try: + results = await self.api_tool_searcher.search( + query=query, + environment=environment, + connection_id=connection_id, + precomputed_embedding=precomputed_embedding, + ) + if results: + matched = results[0] + logger.info( + f"API tool match: {matched.name!r} " + f"(confidence={matched.confidence}, cosine={matched.cosine_score:.4f})" + ) + return ClassificationResult( + workflow=WorkflowType.API_TOOL_CALLING, + confidence=matched.cosine_score, + metadata={"matched_endpoint": matched.to_dict()}, + reasoning=( + f"API tool match: {matched.name} " + f"(cosine={matched.cosine_score:.4f}, confidence={matched.confidence})" + ), + ) + except Exception as e: + logger.error(f"API tool classification failed: {e}", exc_info=True) + return None + async def _execute_with_fallback_async( self, workflow: Any, @@ -651,17 +768,23 @@ async def _execute_with_fallback_async( logger.info(f"[{chat_id}] Executing {workflow_name} (non-streaming)") try: - result = await workflow.execute_async(request, context, time_metric) + if self._is_workflow_enabled(start_layer): + result = await workflow.execute_async(request, context, time_metric) - if result is not None: - logger.info(f"[{chat_id}] {workflow_name} handled successfully") - return result + if result is not None: + logger.info(f"[{chat_id}] {workflow_name} handled successfully") + return result - # Implement layer-wise fallback chain - logger.info( - f"[{chat_id}] {workflow_name} returned None, " - f"trying next layer in fallback chain" - ) + # Implement layer-wise fallback chain + logger.info( + f"[{chat_id}] {workflow_name} returned None, " + f"trying next layer in fallback chain" + ) + else: + logger.info( + f"[{chat_id}] {workflow_name} is disabled via feature flag, " + f"trying next layer in fallback chain" + ) # Get the layer order starting from current layer @@ -670,6 +793,11 @@ async def _execute_with_fallback_async( # Try each subsequent layer in order for next_layer in remaining_layers: + if not self._is_workflow_enabled(next_layer): + next_name = WORKFLOW_DISPLAY_NAMES.get(next_layer, next_layer.value) + logger.info(f"[{chat_id}] Skipping disabled workflow: {next_name}") + continue + next_workflow = self._get_workflow_executor(next_layer) next_name = WORKFLOW_DISPLAY_NAMES.get(next_layer, next_layer.value) @@ -729,19 +857,25 @@ async def _execute_with_fallback_streaming( logger.info(f"[{chat_id}] Executing {workflow_name} (streaming)") try: - result = await workflow.execute_streaming(request, context, time_metric) + if self._is_workflow_enabled(start_layer): + result = await workflow.execute_streaming(request, context, time_metric) - if result is not None: - logger.info(f"[{chat_id}] {workflow_name} streaming started") - async for chunk in result: - yield chunk - return + if result is not None: + logger.info(f"[{chat_id}] {workflow_name} streaming started") + async for chunk in result: + yield chunk + return - # Implement layer-wise fallback chain for streaming - logger.info( - f"[{chat_id}] {workflow_name} returned None, " - f"trying next layer in fallback chain" - ) + # Implement layer-wise fallback chain for streaming + logger.info( + f"[{chat_id}] {workflow_name} returned None, " + f"trying next layer in fallback chain" + ) + else: + logger.info( + f"[{chat_id}] {workflow_name} is disabled via feature flag, " + f"trying next layer in fallback chain" + ) # Get the layer order starting from current layer @@ -750,6 +884,11 @@ async def _execute_with_fallback_streaming( # Try each subsequent layer in order for next_layer in remaining_layers: + if not self._is_workflow_enabled(next_layer): + next_name = WORKFLOW_DISPLAY_NAMES.get(next_layer, next_layer.value) + logger.info(f"[{chat_id}] Skipping disabled workflow: {next_name}") + continue + next_workflow = self._get_workflow_executor(next_layer) next_name = WORKFLOW_DISPLAY_NAMES.get(next_layer, next_layer.value) diff --git a/src/tool_classifier/constants.py b/src/tool_classifier/constants.py index 9628a921..98181afe 100644 --- a/src/tool_classifier/constants.py +++ b/src/tool_classifier/constants.py @@ -109,6 +109,28 @@ Ensures the top result is significantly better than the runner-up.""" +# ============================================================================ +# API Tool Collection Search Configuration +# ============================================================================ + +API_TOOL_COLLECTION = "api_tool_collection" +"""Qdrant collection name for API endpoint semantic search.""" + +API_TOOL_SEARCH_TOP_K = 5 +"""Number of top endpoints to return from API tool semantic search.""" + +API_TOOL_MIN_THRESHOLD = 0.40 +"""Minimum dense cosine similarity to consider a result as an API tool match. +Below this → no API tool matched, fall through to other workflows.""" + +API_TOOL_HIGH_CONFIDENCE_THRESHOLD = 0.60 +"""Dense cosine similarity for high-confidence API tool match. +Above this AND score gap is large → route to API Tool Calling without further LLM disambiguation.""" + +API_TOOL_SCORE_GAP_THRESHOLD = 0.05 +"""Cosine score gap (top - second) for high-confidence API tool classification.""" + + # ============================================================================ # Agentic Loop — Continuation Threshold # ============================================================================ diff --git a/src/tool_classifier/enums.py b/src/tool_classifier/enums.py index df8ced7f..f734bc7d 100644 --- a/src/tool_classifier/enums.py +++ b/src/tool_classifier/enums.py @@ -11,12 +11,14 @@ class WorkflowType(Enum): workflow should handle each user query: - SERVICE: External service/API calls (Layer 1) - - CONTEXT: Conversation history or greetings (Layer 2) - - RAG: Knowledge base retrieval (Layer 3) - - OOD: Out-of-domain fallback (Layer 4) + - API_TOOL_CALLING: External API tool calling via agentic loop (Layer 2) + - CONTEXT: Conversation history or greetings (Layer 3) + - RAG: Knowledge base retrieval (Layer 4) + - OOD: Out-of-domain fallback (Layer 5) """ SERVICE = "service" + API_TOOL_CALLING = "api_tool_calling" CONTEXT = "context" RAG = "rag" OOD = "ood" @@ -25,14 +27,16 @@ class WorkflowType(Enum): # Layer configuration - defines the order of workflow evaluation WORKFLOW_LAYER_ORDER = [ WorkflowType.SERVICE, # Layer 1: Try service first - WorkflowType.CONTEXT, # Layer 2: Then context - WorkflowType.RAG, # Layer 3: Then RAG - WorkflowType.OOD, # Layer 4: Finally OOD (always succeeds) + WorkflowType.API_TOOL_CALLING, # Layer 2: Try API tool calling + WorkflowType.CONTEXT, # Layer 3: Then context + WorkflowType.RAG, # Layer 4: Then RAG + WorkflowType.OOD, # Layer 5: Finally OOD (always succeeds) ] # Workflow display names for logging WORKFLOW_DISPLAY_NAMES = { WorkflowType.SERVICE: "Service Workflow", + WorkflowType.API_TOOL_CALLING: "API Tool Calling Workflow", WorkflowType.CONTEXT: "Context Workflow", WorkflowType.RAG: "RAG Workflow", WorkflowType.OOD: "Out-of-Domain Workflow", diff --git a/src/tool_classifier/workflows/__init__.py b/src/tool_classifier/workflows/__init__.py index 3d733d54..01cab0d2 100644 --- a/src/tool_classifier/workflows/__init__.py +++ b/src/tool_classifier/workflows/__init__.py @@ -1,11 +1,13 @@ """Workflow executor implementations.""" +from tool_classifier.workflows.api_tool_workflow import APIToolWorkflowExecutor from tool_classifier.workflows.service_workflow import ServiceWorkflowExecutor from tool_classifier.workflows.context_workflow import ContextWorkflowExecutor from tool_classifier.workflows.rag_workflow import RAGWorkflowExecutor from tool_classifier.workflows.ood_workflow import OODWorkflowExecutor __all__ = [ + "APIToolWorkflowExecutor", "ServiceWorkflowExecutor", "ContextWorkflowExecutor", "RAGWorkflowExecutor", diff --git a/src/tool_classifier/workflows/api_tool_workflow.py b/src/tool_classifier/workflows/api_tool_workflow.py new file mode 100644 index 00000000..9403a10a --- /dev/null +++ b/src/tool_classifier/workflows/api_tool_workflow.py @@ -0,0 +1,136 @@ +"""API Tool Calling Workflow Executor — Layer 2 of the classification chain. + +This is the Task 4.1 minimal implementation that surfaces the matched API endpoint +to the user. The full agentic loop (parameter collection → API call → response +formatting) will be implemented in Task 10. +""" + +from typing import Any, AsyncIterator, Dict, Optional + +from loguru import logger + +from models.request_models import OrchestrationRequest, OrchestrationResponse +from tool_classifier.base_workflow import BaseWorkflow + + +class APIToolWorkflowExecutor(BaseWorkflow): + """Executes API Tool Calling workflow (Layer 2). + + Handles queries that matched an API endpoint in api_tool_collection. + Reads the matched endpoint from context (populated by ToolClassifier.classify()) + and returns it as a response. + + Task 10 will replace the placeholder response body with the full agentic loop: + session management → parameter collection → external API call → response formatting. + """ + + def __init__(self, orchestration_service: Optional[Any] = None) -> None: + """Initialize API tool calling workflow. + + Args: + orchestration_service: Reference to LLMOrchestrationService — required + for streaming mode (format_sse). Optional for non-streaming. + """ + self.orchestration_service = orchestration_service + + async def execute_async( + self, + request: OrchestrationRequest, + context: Dict[str, Any], + time_metric: Optional[Dict[str, float]] = None, + ) -> Optional[OrchestrationResponse]: + """Execute API tool calling workflow in non-streaming mode. + + Args: + request: Orchestration request. + context: Must contain "matched_endpoint" dict from APISemanticSearcher. + time_metric: Optional timing dict for step tracking. + + Returns: + OrchestrationResponse with matched endpoint info, or None if no + endpoint in context (triggers fallback to next layer). + """ + chat_id = request.chatId + endpoint = context.get("matched_endpoint") + + if not endpoint: + logger.warning( + f"[{chat_id}] APIToolWorkflow: no matched_endpoint in context — falling back" + ) + return None + + name = endpoint.get("name", "unknown") + description = endpoint.get("description", "") + url = endpoint.get("url", "N/A") + confidence = endpoint.get("confidence", "medium") + cosine_score = endpoint.get("cosine_score", 0.0) + + logger.info( + f"[{chat_id}] APIToolWorkflow: matched endpoint={name!r} " + f"(confidence={confidence}, cosine={cosine_score:.4f})" + ) + + # TODO (Task 10): Replace with full agentic loop — + # load/create session → run param extraction → call external API → format response + content = f"**{name}**: {description}\n\nURL: {url}" + + return OrchestrationResponse( + chatId=chat_id, + llmServiceActive=True, + questionOutOfLLMScope=False, + inputGuardFailed=False, + content=content, + ) + + async def execute_streaming( + self, + request: OrchestrationRequest, + context: Dict[str, Any], + time_metric: Optional[Dict[str, float]] = None, + ) -> Optional[AsyncIterator[str]]: + """Execute API tool calling workflow in streaming mode. + + Args: + request: Orchestration request. + context: Must contain "matched_endpoint" dict from APISemanticSearcher. + time_metric: Optional timing dict for step tracking. + + Returns: + AsyncIterator yielding SSE-formatted strings, or None on failure. + """ + chat_id = request.chatId + endpoint = context.get("matched_endpoint") + + if not endpoint: + logger.warning( + f"[{chat_id}] APIToolWorkflow streaming: no matched_endpoint — falling back" + ) + return None + + if self.orchestration_service is None: + logger.error( + f"[{chat_id}] APIToolWorkflow streaming: orchestration_service not set" + ) + return None + + name = endpoint.get("name", "unknown") + description = endpoint.get("description", "") + url = endpoint.get("url", "N/A") + confidence = endpoint.get("confidence", "medium") + cosine_score = endpoint.get("cosine_score", 0.0) + + logger.info( + f"[{chat_id}] APIToolWorkflow streaming: matched endpoint={name!r} " + f"(confidence={confidence}, cosine={cosine_score:.4f})" + ) + + # TODO (Task 10): Replace with full agentic loop streaming + content = f"**{name}**: {description}\n\nURL: {url}" + + orchestration_service = self.orchestration_service + + async def _stream() -> AsyncIterator[str]: + yield orchestration_service.format_sse(chat_id, content) + yield orchestration_service.format_sse(chat_id, "END") + + return _stream() diff --git a/tests/api_tool_eval/batch_index.py b/tests/api_tool_eval/batch_index.py new file mode 100644 index 00000000..bff78958 --- /dev/null +++ b/tests/api_tool_eval/batch_index.py @@ -0,0 +1,106 @@ +""" +Batch Indexer — sends all endpoints from endpoints.json to POST /api-tools/index. + +Usage: + python batch_index.py + python batch_index.py --ruuter-url http://localhost:8086 + python batch_index.py --skip-existing # skip endpoints already in Qdrant +""" + +import argparse +import json +import time +from pathlib import Path + +import requests + +ENDPOINTS_FILE = Path(__file__).parent / "endpoints.json" +DEFAULT_RUUTER_URL = "http://localhost:8086" +INDEX_ENDPOINT = "/rag-search/api-tools/index" + + +def index_endpoint(ruuter_url: str, endpoint: dict) -> dict: + """Send a single endpoint spec to the indexing API.""" + url = f"{ruuter_url}{INDEX_ENDPOINT}" + try: + response = requests.post(url, json=endpoint, timeout=60) + return { + "status_code": response.status_code, + "body": response.json() if response.content else {}, + "ok": 200 <= response.status_code < 300, + } + except requests.exceptions.Timeout: + return {"status_code": 408, "body": {"error": "Timeout"}, "ok": False} + except requests.exceptions.ConnectionError as e: + return {"status_code": 503, "body": {"error": str(e)}, "ok": False} + except Exception as e: + return {"status_code": 500, "body": {"error": str(e)}, "ok": False} + + +def main(): + parser = argparse.ArgumentParser( + description="Batch index API endpoints into Qdrant" + ) + parser.add_argument( + "--ruuter-url", + default=DEFAULT_RUUTER_URL, + help=f"Base URL of Ruuter public (default: {DEFAULT_RUUTER_URL})", + ) + parser.add_argument( + "--delay", + type=float, + default=1.0, + help="Seconds to wait between indexing requests (default: 1.0)", + ) + args = parser.parse_args() + + # Load endpoints + if not ENDPOINTS_FILE.exists(): + print(f"ERROR: {ENDPOINTS_FILE} not found") + return + + with open(ENDPOINTS_FILE, encoding="utf-8") as f: + endpoints = json.load(f) + + print(f"\n{'=' * 60}") + print(f"Batch Indexer — {len(endpoints)} endpoints") + print(f"Target: {args.ruuter_url}{INDEX_ENDPOINT}") + print(f"{'=' * 60}\n") + + results = {"success": [], "failed": []} + + for i, endpoint in enumerate(endpoints, 1): + name = endpoint.get("name", "unknown") + endpoint_id = endpoint.get("endpointId", "?") + print(f"[{i:02d}/{len(endpoints)}] Indexing: {name} ({endpoint_id[:8]}...)") + + result = index_endpoint(args.ruuter_url, endpoint) + + if result["ok"]: + print(f" Success (HTTP {result['status_code']})") + results["success"].append(name) + else: + print( + f" Failed (HTTP {result['status_code']}) — {result['body']}" + ) + results["failed"].append(name) + + # Delay between requests to avoid overloading the embedding service + if i < len(endpoints): + time.sleep(args.delay) + + # Summary + print(f"\n{'=' * 60}") + print("INDEXING COMPLETE") + print(f"{'=' * 60}") + print(f" Success: {len(results['success'])}/{len(endpoints)}") + print(f" Failed: {len(results['failed'])}/{len(endpoints)}") + if results["failed"]: + print("\nFailed endpoints:") + for name in results["failed"]: + print(f" - {name}") + print("\nNext step: run python eval_search.py to test retrieval accuracy") + + +if __name__ == "__main__": + main() diff --git a/tests/api_tool_eval/endpoints.json b/tests/api_tool_eval/endpoints.json new file mode 100644 index 00000000..52d1f981 --- /dev/null +++ b/tests/api_tool_eval/endpoints.json @@ -0,0 +1,223 @@ +[ + { + "endpointId": "a3f7c2d1-84e6-4b19-92f3-d51c7e890ab2", + "name": "get_national_holidays", + "description": "Fetch national holidays for a specific country to see when they have public days off.", + "url": "https://openholidaysapi.org/PublicHolidays", + "method": "GET", + "params": [ + { "name": "countryIsoCode", "type": "string", "required": true, "description": "The 2-letter ISO country code (e.g., EE for Estonia, DE for Germany)" }, + { "name": "languageIsoCode", "type": "string", "required": false, "description": "The 2-letter ISO language code for the response (e.g., ET, EN)" }, + { "name": "validFrom", "type": "date", "required": false, "description": "Start date for the holiday search (YYYY-MM-DD)" }, + { "name": "validTo", "type": "date", "required": false, "description": "End date for the holiday search (YYYY-MM-DD)" } + ] + }, + { + "endpointId": "c6d092b4-f518-4a37-b9e7-120e83a7d64f", + "name": "get_unemployment_rate", + "description": "Fetch the official unemployment rate statistics for Estonia from the national statistics office.", + "url": "https://andmed.stat.ee/api/v1/en/stat/TT330", + "method": "POST", + "params": [ + { "name": "Näitaja", "type": "string", "required": true, "description": "The specific statistical indicator to query (e.g., unemployment rate)" }, + { "name": "Sugu", "type": "string", "required": false, "description": "Gender filter (e.g., Male, Female, Total)" }, + { "name": "Vanuserühm", "type": "string", "required": false, "description": "Age group category filter (e.g., 15-24, 25-49)" }, + { "name": "Vaatlusperiood", "type": "string", "required": true, "description": "The observation period (e.g., year or quarter) to get data for" } + ] + }, + { + "endpointId": "b8e41f09-3c72-4d85-ae16-7f924d10c53e", + "name": "get_current_electricity_price", + "description": "Fetch the current real-time electricity and energy market price for Estonia.", + "url": "https://dashboard.elering.ee/api/nps/price/EE/current", + "method": "GET", + "params": [] + }, + { + "endpointId": "d1e2f3a4-b5c6-7890-abcd-ef1234567890", + "name": "get_weather_forecast", + "description": "Fetch hourly or daily weather forecast for any location including temperature, precipitation, and wind speed.", + "url": "https://api.open-meteo.com/v1/forecast", + "method": "GET", + "params": [ + { "name": "latitude", "type": "float", "required": true, "description": "Geographic latitude of the location (e.g., 59.4370 for Tallinn)" }, + { "name": "longitude", "type": "float", "required": true, "description": "Geographic longitude of the location (e.g., 24.7536 for Tallinn)" }, + { "name": "hourly", "type": "string", "required": false, "description": "Hourly weather variables to include (e.g., temperature_2m, precipitation, windspeed_10m)" }, + { "name": "daily", "type": "string", "required": false, "description": "Daily weather variables to include (e.g., temperature_2m_max, precipitation_sum)" }, + { "name": "forecast_days", "type": "integer", "required": false, "description": "Number of forecast days (1-16)" } + ] + }, + { + "endpointId": "e2f3a4b5-c6d7-8901-bcde-f12345678901", + "name": "get_exchange_rates", + "description": "Fetch the latest foreign currency exchange rates relative to a base currency.", + "url": "https://api.frankfurter.app/latest", + "method": "GET", + "params": [ + { "name": "base", "type": "string", "required": false, "description": "Base currency code to convert from (e.g., EUR, USD). Defaults to EUR." }, + { "name": "symbols", "type": "string", "required": false, "description": "Comma-separated list of target currency codes to include (e.g., USD,GBP,SEK)" } + ] + }, + { + "endpointId": "f3a4b5c6-d7e8-9012-cdef-123456789012", + "name": "get_country_information", + "description": "Fetch detailed country information including population, capital city, languages, currency, and geographic data.", + "url": "https://restcountries.com/v3.1/name", + "method": "GET", + "params": [ + { "name": "country", "type": "string", "required": true, "description": "Full or partial name of the country (e.g., Estonia, Germany, Finland)" }, + { "name": "fields", "type": "string", "required": false, "description": "Comma-separated fields to return (e.g., name,capital,population,currencies)" } + ] + }, + { + "endpointId": "a4b5c6d7-e8f9-0123-defa-234567890123", + "name": "get_ip_geolocation", + "description": "Fetch geographic location information for an IP address including country, city, region, and coordinates.", + "url": "http://ip-api.com/json", + "method": "GET", + "params": [ + { "name": "ip", "type": "string", "required": true, "description": "The IP address to geolocate (e.g., 88.196.123.45)" }, + { "name": "fields", "type": "string", "required": false, "description": "Comma-separated fields to return (e.g., country,city,lat,lon,isp)" } + ] + }, + { + "endpointId": "b5c6d7e8-f9a0-1234-efab-345678901234", + "name": "get_school_holidays", + "description": "Fetch official school holidays and term breaks for a specific country and school year.", + "url": "https://openholidaysapi.org/SchoolHolidays", + "method": "GET", + "params": [ + { "name": "countryIsoCode", "type": "string", "required": true, "description": "The 2-letter ISO country code (e.g., EE for Estonia)" }, + { "name": "languageIsoCode", "type": "string", "required": false, "description": "The 2-letter ISO language code for the response (e.g., ET, EN)" }, + { "name": "validFrom", "type": "date", "required": false, "description": "Start date for the school holiday search (YYYY-MM-DD)" }, + { "name": "validTo", "type": "date", "required": false, "description": "End date for the school holiday search (YYYY-MM-DD)" } + ] + }, + { + "endpointId": "c6d7e8f9-a0b1-2345-fabc-456789012345", + "name": "get_current_time_by_timezone", + "description": "Fetch the current time, date, UTC offset, and daylight saving time status for any timezone.", + "url": "https://worldtimeapi.org/api/timezone", + "method": "GET", + "params": [ + { "name": "timezone", "type": "string", "required": true, "description": "IANA timezone name (e.g., Europe/Tallinn, Europe/Helsinki, UTC)" } + ] + }, + { + "endpointId": "d7e8f9a0-b1c2-3456-abcd-567890123456", + "name": "get_air_quality", + "description": "Fetch real-time air quality measurements including PM2.5, PM10, NO2, ozone, and AQI index for a location.", + "url": "https://api.openaq.org/v2/latest", + "method": "GET", + "params": [ + { "name": "city", "type": "string", "required": false, "description": "City name to filter air quality sensors (e.g., Tallinn, Tartu)" }, + { "name": "country", "type": "string", "required": false, "description": "2-letter ISO country code to filter sensors (e.g., EE)" }, + { "name": "parameter", "type": "string", "required": false, "description": "Pollutant to filter by (e.g., pm25, pm10, no2, o3)" }, + { "name": "limit", "type": "integer", "required": false, "description": "Maximum number of results to return" } + ] + }, + { + "endpointId": "e8f9a0b1-c2d3-4567-bcde-678901234567", + "name": "get_address_geocoding", + "description": "Search for a geographic address and return its latitude, longitude, and structured address components.", + "url": "https://nominatim.openstreetmap.org/search", + "method": "GET", + "params": [ + { "name": "q", "type": "string", "required": true, "description": "Free-form address or place name to search for (e.g., Viru 4, Tallinn)" }, + { "name": "format", "type": "string", "required": false, "description": "Response format: json, xml, geojson. Default: json" }, + { "name": "countrycodes", "type": "string", "required": false, "description": "Comma-separated ISO country codes to restrict search (e.g., ee for Estonia)" }, + { "name": "limit", "type": "integer", "required": false, "description": "Maximum number of results to return (default: 10)" } + ] + }, + { + "endpointId": "f9a0b1c2-d3e4-5678-cdef-789012345678", + "name": "get_word_definition", + "description": "Fetch the definition, phonetics, synonyms, antonyms, and usage examples for an English word.", + "url": "https://api.dictionaryapi.dev/api/v2/entries/en", + "method": "GET", + "params": [ + { "name": "word", "type": "string", "required": true, "description": "The English word to look up the definition for (e.g., ephemeral, resilient)" } + ] + }, + { + "endpointId": "a0b1c2d3-e4f5-6789-defa-890123456789", + "name": "get_gdp_statistics", + "description": "Fetch annual GDP and economic growth rate statistics for any country from the World Bank open data API.", + "url": "https://api.worldbank.org/v2/country/indicator/NY.GDP.MKTP.CD", + "method": "GET", + "params": [ + { "name": "country", "type": "string", "required": true, "description": "ISO 2-letter country code (e.g., EE for Estonia, FI for Finland)" }, + { "name": "format", "type": "string", "required": false, "description": "Response format: json or xml. Default: json" }, + { "name": "date", "type": "string", "required": false, "description": "Year or year range to filter (e.g., 2020 or 2015:2023)" }, + { "name": "per_page", "type": "integer", "required": false, "description": "Number of records per page (default: 50)" } + ] + }, + { + "endpointId": "b1c2d3e4-f5a6-7890-efab-901234567890", + "name": "get_population_data", + "description": "Fetch annual population count and growth statistics for any country from the World Bank.", + "url": "https://api.worldbank.org/v2/country/indicator/SP.POP.TOTL", + "method": "GET", + "params": [ + { "name": "country", "type": "string", "required": true, "description": "ISO 2-letter country code (e.g., EE for Estonia)" }, + { "name": "format", "type": "string", "required": false, "description": "Response format: json or xml. Default: json" }, + { "name": "date", "type": "string", "required": false, "description": "Year or year range (e.g., 2020 or 2010:2023)" } + ] + }, + { + "endpointId": "c2d3e4f5-a6b7-8901-fabc-012345678901", + "name": "get_electricity_price_history", + "description": "Fetch historical electricity market prices for Estonia over a specified date range.", + "url": "https://dashboard.elering.ee/api/nps/price", + "method": "GET", + "params": [ + { "name": "start", "type": "datetime", "required": true, "description": "Start datetime for the price history query (ISO 8601 format, e.g., 2024-01-01T00:00:00Z)" }, + { "name": "end", "type": "datetime", "required": true, "description": "End datetime for the price history query (ISO 8601 format, e.g., 2024-01-31T23:59:59Z)" } + ] + }, + { + "endpointId": "e4f5a6b7-c8d9-0123-bcde-234567890123", + "name": "get_public_transport_stops", + "description": "Fetch list of public transport stops in Estonia including buses, trams, and trains with their GPS coordinates.", + "url": "https://peatus.ee/gtfs/stops.txt", + "method": "GET", + "params": [] + }, + { + "endpointId": "f5a6b7c8-d9e0-1234-cdef-345678901234", + "name": "get_average_salary_statistics", + "description": "Fetch official average gross and net salary statistics for Estonia by sector, region, or time period from Statistics Estonia.", + "url": "https://andmed.stat.ee/api/v1/en/stat/PA5321", + "method": "POST", + "params": [ + { "name": "Aasta", "type": "string", "required": true, "description": "Year of the salary data to retrieve (e.g., 2023, 2022)" }, + { "name": "Tegevusala", "type": "string", "required": false, "description": "Industry or economic sector to filter by (e.g., total economy, manufacturing, IT)" }, + { "name": "Maakond", "type": "string", "required": false, "description": "Estonian county or region to filter by (e.g., Harju, Tartu)" } + ] + }, + { + "endpointId": "a6b7c8d9-e0f1-2345-defa-456789012345", + "name": "get_estonian_company_info", + "description": "Fetch official company registration details from the Estonian Business Registry including name, registration number, address, legal status, and board members.", + "url": "https://ariregister.rik.ee/api/v1/company", + "method": "GET", + "params": [ + { "name": "reg_code", "type": "string", "required": false, "description": "Company registration number to look up (e.g., 10000000)" }, + { "name": "name", "type": "string", "required": false, "description": "Company name or partial name to search for" }, + { "name": "status", "type": "string", "required": false, "description": "Filter by company status: active, liquidated, bankrupt" } + ] + }, + { + "endpointId": "b7c8d9e0-f1a2-3456-efab-567890123456", + "name": "get_reverse_geocoding", + "description": "Convert GPS coordinates (latitude and longitude) into a human-readable street address or place name.", + "url": "https://nominatim.openstreetmap.org/reverse", + "method": "GET", + "params": [ + { "name": "lat", "type": "float", "required": true, "description": "Latitude of the location to reverse geocode (e.g., 59.4370 for Tallinn)" }, + { "name": "lon", "type": "float", "required": true, "description": "Longitude of the location to reverse geocode (e.g., 24.7536 for Tallinn)" }, + { "name": "format", "type": "string", "required": false, "description": "Response format: json or xml. Default: json" }, + { "name": "zoom", "type": "integer", "required": false, "description": "Level of detail for the address (3=country, 10=city, 18=building)" } + ] + } +] \ No newline at end of file diff --git a/tests/api_tool_eval/eval_search.py b/tests/api_tool_eval/eval_search.py new file mode 100644 index 00000000..4fcf5eff --- /dev/null +++ b/tests/api_tool_eval/eval_search.py @@ -0,0 +1,383 @@ +""" +Retrieval Evaluation Script — tests semantic search accuracy across 40+ queries. + +Usage: + python eval_search.py + python eval_search.py --ruuter-url http://localhost:8086 + python eval_search.py --output results.json # also save detailed JSON results + +What it does: + 1. Sends each query to POST /rag-search/api-tools/search + 2. Checks if the top result matches the expected endpoint name + 3. Prints a detailed pass/fail table + 4. Outputs accuracy %, average cosine score, and a list of failures +""" + +import argparse +import json +import time +from pathlib import Path +from typing import Optional + +import requests + +DEFAULT_RUUTER_URL = "http://localhost:8086" +SEARCH_ENDPOINT = "/rag-search/api-tools/search" + +# ============================================================================ +# Evaluation Dataset +# Format: (query, expected_endpoint_name or None for "no match expected") +# ============================================================================ +EVAL_QUERIES = [ + # --- get_national_holidays --- + ("What are the national holidays in Estonia?", "get_national_holidays"), + ( + "What are the upcoming national holidays in Estonia this year?", + "get_national_holidays", + ), + ("List all public days off in Estonia this year", "get_national_holidays"), + ("Show me national holidays for Estonia", "get_national_holidays"), + ("What are the official Estonian public holidays?", "get_national_holidays"), + # --- get_school_holidays --- + ("When are the school holidays in Estonia?", "get_school_holidays"), + ("What are the school term breaks in Estonia?", "get_school_holidays"), + ("When does school summer break start in Estonia?", "get_school_holidays"), + # --- get_current_electricity_price --- + ( + "What is the current electricity price in Estonia?", + "get_current_electricity_price", + ), + ( + "How much does electricity cost right now in Estonia?", + "get_current_electricity_price", + ), + ( + "Show me the real-time energy market price in Estonia", + "get_current_electricity_price", + ), + ( + "What is the spot price for electricity in Estonia today?", + "get_current_electricity_price", + ), + # --- get_electricity_price_history --- + ( + "Show me the electricity price history for Estonia over the last month", + "get_electricity_price_history", + ), + ( + "Show me historical electricity prices for Estonia in January 2024", + "get_electricity_price_history", + ), + ( + "Fetch the electricity price history for Estonia for the past 30 days", + "get_electricity_price_history", + ), + # --- get_unemployment_rate --- + ("What is the unemployment rate in Estonia?", "get_unemployment_rate"), + ("How many people are unemployed in Estonia this year?", "get_unemployment_rate"), + ("Show me the latest jobless statistics for Estonia", "get_unemployment_rate"), + ("What percentage of Estonians are unemployed?", "get_unemployment_rate"), + # --- get_weather_forecast --- + ("What is the weather forecast for Tallinn tomorrow?", "get_weather_forecast"), + ("What is the weather forecast for Tartu next week?", "get_weather_forecast"), + ( + "What is the weather forecast and temperature for Pärnu this weekend?", + "get_weather_forecast", + ), + ("Show me the 7-day weather forecast for Tallinn", "get_weather_forecast"), + ( + "What are the weather conditions including wind speed in Narva today?", + "get_weather_forecast", + ), + # --- get_exchange_rates --- + ("What is the EUR to USD exchange rate today?", "get_exchange_rates"), + ("Show me the current currency exchange rates", "get_exchange_rates"), + ("What is the exchange rate from EUR to Swedish krona?", "get_exchange_rates"), + ("What are the latest forex rates for EUR?", "get_exchange_rates"), + # --- get_country_information --- + ( + "Get country information for Estonia including its capital city", + "get_country_information", + ), + ( + "What country information is available for Estonia, including official languages?", + "get_country_information", + ), + ("Fetch country details and facts about Estonia", "get_country_information"), + ("What is the country profile for Estonia?", "get_country_information"), + # --- get_ip_geolocation --- + ("What is the geolocation of IP address 88.196.123.45?", "get_ip_geolocation"), + ( + "Geolocate this IP address and find which country it belongs to", + "get_ip_geolocation", + ), + ("Find the geolocation of an IP address", "get_ip_geolocation"), + # --- get_current_time_by_timezone --- + ("What time is it in Tallinn right now?", "get_current_time_by_timezone"), + ( + "What is the current time in the Europe/Tallinn timezone?", + "get_current_time_by_timezone", + ), + ( + "What is the current time in Estonia and is it in daylight saving timezone?", + "get_current_time_by_timezone", + ), + # --- get_air_quality --- + ("What is the air quality in Tallinn today?", "get_air_quality"), + ("Show me PM2.5 pollution levels in Tallinn", "get_air_quality"), + ("Is the air quality good in Tartu right now?", "get_air_quality"), + # --- get_address_geocoding --- + ("Find the coordinates for Viru 4, Tallinn", "get_address_geocoding"), + ( + "Get the geocoding coordinates for Kadriorg Park in Tallinn", + "get_address_geocoding", + ), + ( + "What are the GPS coordinates of this address in Estonia?", + "get_address_geocoding", + ), + # --- get_gdp_statistics --- + ("What is Estonia's GDP this year?", "get_gdp_statistics"), + ("What is the economic output of Estonia?", "get_gdp_statistics"), + ( + "Show me the GDP growth rate of Estonia over the past 5 years", + "get_gdp_statistics", + ), + # --- get_population_data --- + ("What is the total population of Estonia?", "get_population_data"), + ("What is the total population data for Estonia?", "get_population_data"), + ("What is the population growth rate of Estonia?", "get_population_data"), + # --- get_word_definition --- + ("Get the word definition for ephemeral", "get_word_definition"), + ("Look up the word definition for resilient", "get_word_definition"), + ("Fetch the dictionary definition of the word sustainable", "get_word_definition"), + # --- get_public_transport_stops --- + ("Where are the bus stops in Tallinn?", "get_public_transport_stops"), + ( + "Show me public transport stops near Tartu city centre", + "get_public_transport_stops", + ), + # --- get_average_salary_statistics --- + ("What is the average salary in Estonia?", "get_average_salary_statistics"), + ("How much do people earn in Estonia on average?", "get_average_salary_statistics"), + ( + "What is the average monthly wage in the IT sector in Estonia?", + "get_average_salary_statistics", + ), + # --- get_estonian_company_info --- + ( + "Look up company registration number 10000000 in Estonia", + "get_estonian_company_info", + ), + ( + "Find details about an Estonian company called Tallinn IT OÜ", + "get_estonian_company_info", + ), + ( + "Is this Estonian company still active in the business registry?", + "get_estonian_company_info", + ), + # --- get_reverse_geocoding --- + ( + "Reverse geocode the coordinates 59.4370, 24.7536 to get the street address", + "get_reverse_geocoding", + ), + ("Convert GPS coordinates to a street address in Tallinn", "get_reverse_geocoding"), + ( + "Reverse geocoding for latitude 58.3780 longitude 26.7290 in Tartu", + "get_reverse_geocoding", + ), + # --- NEGATIVE queries — should return NO matching results --- + ("Who is the Prime Minister of Estonia?", None), + ("What is the best restaurant in Tallinn?", None), + ("Tell me a random fact about Estonia", None), + ("What is the meaning of life?", None), + ("Book me a flight to London", None), + ("Can you translate this text to Estonian?", None), + ("What are the visa requirements to visit Estonia?", None), + ("How do I apply for an Estonian e-Residency?", None), + ("What is the history of Tallinn Old Town?", None), + ("Give me a poem about Estonia", None), +] + + +def search(ruuter_url: str, query: str, top_k: int = 3) -> Optional[dict]: + """Send a search query and return the parsed response.""" + url = f"{ruuter_url}{SEARCH_ENDPOINT}" + try: + response = requests.post( + url, + json={"query": query, "top_k": top_k, "environment": "production"}, + timeout=30, + ) + if response.status_code == 200: + body = response.json() + # Handle Ruuter wrapper: body may be {"response": {...}} + return body.get("response", body) + return None + except Exception: + return None + + +def evaluate(ruuter_url: str, delay: float = 0.5) -> list: + """Run all evaluation queries and return results.""" + results = [] + for query, expected in EVAL_QUERIES: + response = search(ruuter_url, query) + + if response is None: + results.append( + { + "query": query, + "expected": expected, + "got": "ERROR", + "cosine_score": None, + "confidence": None, + "pass": False, + "error": "Request failed", + } + ) + time.sleep(delay) + continue + + top_results = response.get("results", []) + top = top_results[0] if top_results else None + + got_name = top["name"] if top else None + cosine_score = top["cosine_score"] if top else None + rrf_score = top["rrf_score"] if top else None + confidence = top["confidence"] if top else None + + # Determine pass/fail + if expected is None: + # Negative query: should return no HIGH confidence result + passed = got_name is None or confidence != "high" + else: + passed = got_name == expected + + results.append( + { + "query": query, + "expected": expected, + "got": got_name, + "cosine_score": cosine_score, + "rrf_score": rrf_score, + "confidence": confidence, + "pass": passed, + } + ) + + time.sleep(delay) + + return results + + +def print_report(results: list) -> None: + """Print evaluation results table and summary.""" + PASS = "✅" + FAIL = "❌" + SKIP = "⚠️ " + + print(f"\n{'=' * 100}") + print(f"{'RETRIEVAL EVALUATION REPORT':^100}") + print(f"{'=' * 100}") + print( + f"{'#':<4} {'Query':<48} {'Expected':<28} {'Got':<28} {'Cosine':>7} {'RRF':>9} {'Result'}" + ) + print(f"{'-' * 100}") + + for i, r in enumerate(results, 1): + query = r["query"][:46] + ".." if len(r["query"]) > 46 else r["query"] + expected = (r["expected"] or "(none)")[:26] + got = (r["got"] or "(none)")[:26] + cosine = ( + f"{r['cosine_score']:.4f}" if r["cosine_score"] is not None else " - " + ) + rrf = f"{r['rrf_score']:.6f}" if r["rrf_score"] is not None else " - " + verdict = PASS if r["pass"] else FAIL + + # Highlight negative query failures + if r["expected"] is None and r["got"] is not None and r["confidence"] == "high": + verdict = FAIL + " FALSE POSITIVE" + + print( + f"{i:<4} {query:<48} {expected:<28} {got:<28} {cosine:>7} {rrf:>9} {verdict}" + ) + + # Summary stats + total = len(results) + passed = sum(1 for r in results if r["pass"]) + failed = total - passed + + positives = [r for r in results if r["expected"] is not None] + negatives = [r for r in results if r["expected"] is None] + positive_pass = sum(1 for r in positives if r["pass"]) + negative_pass = sum(1 for r in negatives if r["pass"]) + + scores = [ + r["cosine_score"] + for r in results + if r["cosine_score"] is not None and r["pass"] + ] + avg_cosine = sum(scores) / len(scores) if scores else 0.0 + rrf_scores = [ + r["rrf_score"] for r in results if r["rrf_score"] is not None and r["pass"] + ] + avg_rrf = sum(rrf_scores) / len(rrf_scores) if rrf_scores else 0.0 + + print(f"\n{'=' * 100}") + print("SUMMARY") + print(f"{'=' * 100}") + print(f" Overall Accuracy: {passed}/{total} ({100 * passed / total:.1f}%)") + print( + f" Positive Queries: {positive_pass}/{len(positives)} ({100 * positive_pass / len(positives):.1f}%)" + ) + print( + f" Negative Queries: {negative_pass}/{len(negatives)} ({100 * negative_pass / len(negatives):.1f}%)" + ) + print( + f" Avg Cosine (correct): {avg_cosine:.4f} (threshold: min={0.40}, high={0.60})" + ) + print(f" Avg RRF (correct): {avg_rrf:.6f}") + print("\n Target: >90% overall accuracy, avg cosine >0.55") + + if failed > 0: + print(f"\n FAILURES ({failed}):") + for r in results: + if not r["pass"]: + print(f" '{r['query']}'") + print( + f" Expected: {r['expected']} | Got: {r['got']} | Cosine: {r['cosine_score']} | RRF: {r['rrf_score']}" + ) + + print(f"{'=' * 100}\n") + + +def main(): + parser = argparse.ArgumentParser( + description="Evaluate semantic search retrieval accuracy" + ) + parser.add_argument("--ruuter-url", default=DEFAULT_RUUTER_URL) + parser.add_argument( + "--delay", type=float, default=0.5, help="Seconds between requests" + ) + parser.add_argument( + "--output", type=str, default=None, help="Save results to JSON file" + ) + args = parser.parse_args() + + print(f"\nStarting evaluation — {len(EVAL_QUERIES)} queries") + print(f"Target: {args.ruuter_url}{SEARCH_ENDPOINT}\n") + + results = evaluate(args.ruuter_url, args.delay) + print_report(results) + + if args.output: + output_path = Path(args.output) + with open(output_path, "w", encoding="utf-8") as f: + json.dump(results, f, indent=2) + print(f"Detailed results saved to: {output_path}") + + +if __name__ == "__main__": + main() diff --git a/tests/api_tool_eval/results.json b/tests/api_tool_eval/results.json new file mode 100644 index 00000000..1c089e58 --- /dev/null +++ b/tests/api_tool_eval/results.json @@ -0,0 +1,668 @@ +[ + { + "query": "What are the national holidays in Estonia?", + "expected": "get_national_holidays", + "got": "get_national_holidays", + "cosine_score": 0.4781, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "What are the upcoming national holidays in Estonia this year?", + "expected": "get_national_holidays", + "got": "get_national_holidays", + "cosine_score": 0.4738, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "List all public days off in Estonia this year", + "expected": "get_national_holidays", + "got": "get_national_holidays", + "cosine_score": 0.4398, + "rrf_score": 0.833333, + "confidence": "medium", + "pass": true + }, + { + "query": "Show me national holidays for Estonia", + "expected": "get_national_holidays", + "got": "get_national_holidays", + "cosine_score": 0.5081, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "What are the official Estonian public holidays?", + "expected": "get_national_holidays", + "got": "get_national_holidays", + "cosine_score": 0.4171, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "When are the school holidays in Estonia?", + "expected": "get_school_holidays", + "got": "get_school_holidays", + "cosine_score": 0.4986, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "What are the school term breaks in Estonia?", + "expected": "get_school_holidays", + "got": "get_school_holidays", + "cosine_score": 0.5082, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "When does school summer break start in Estonia?", + "expected": "get_school_holidays", + "got": "get_school_holidays", + "cosine_score": 0.4147, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "What is the current electricity price in Estonia?", + "expected": "get_current_electricity_price", + "got": "get_current_electricity_price", + "cosine_score": 0.7182, + "rrf_score": 1.0, + "confidence": "high", + "pass": true + }, + { + "query": "How much does electricity cost right now in Estonia?", + "expected": "get_current_electricity_price", + "got": "get_current_electricity_price", + "cosine_score": 0.6776, + "rrf_score": 1.0, + "confidence": "high", + "pass": true + }, + { + "query": "Show me the real-time energy market price in Estonia", + "expected": "get_current_electricity_price", + "got": "get_current_electricity_price", + "cosine_score": 0.7339, + "rrf_score": 1.0, + "confidence": "high", + "pass": true + }, + { + "query": "What is the spot price for electricity in Estonia today?", + "expected": "get_current_electricity_price", + "got": "get_current_electricity_price", + "cosine_score": 0.6782, + "rrf_score": 1.0, + "confidence": "high", + "pass": true + }, + { + "query": "Show me the electricity price history for Estonia over the last month", + "expected": "get_electricity_price_history", + "got": "get_electricity_price_history", + "cosine_score": 0.6211, + "rrf_score": 0.833333, + "confidence": "medium", + "pass": true + }, + { + "query": "Show me historical electricity prices for Estonia in January 2024", + "expected": "get_electricity_price_history", + "got": "get_electricity_price_history", + "cosine_score": 0.606, + "rrf_score": 0.833333, + "confidence": "medium", + "pass": true + }, + { + "query": "Fetch the electricity price history for Estonia for the past 30 days", + "expected": "get_electricity_price_history", + "got": "get_electricity_price_history", + "cosine_score": 0.7049, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "What is the unemployment rate in Estonia?", + "expected": "get_unemployment_rate", + "got": "get_unemployment_rate", + "cosine_score": 0.6034, + "rrf_score": 1.0, + "confidence": "high", + "pass": true + }, + { + "query": "How many people are unemployed in Estonia this year?", + "expected": "get_unemployment_rate", + "got": "get_unemployment_rate", + "cosine_score": 0.5537, + "rrf_score": 0.5, + "confidence": "medium", + "pass": true + }, + { + "query": "Show me the latest jobless statistics for Estonia", + "expected": "get_unemployment_rate", + "got": "get_unemployment_rate", + "cosine_score": 0.5786, + "rrf_score": 0.833333, + "confidence": "medium", + "pass": true + }, + { + "query": "What percentage of Estonians are unemployed?", + "expected": "get_unemployment_rate", + "got": "get_unemployment_rate", + "cosine_score": 0.5649, + "rrf_score": 0.5, + "confidence": "medium", + "pass": true + }, + { + "query": "What is the weather forecast for Tallinn tomorrow?", + "expected": "get_weather_forecast", + "got": "get_weather_forecast", + "cosine_score": 0.4984, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "What is the weather forecast for Tartu next week?", + "expected": "get_weather_forecast", + "got": "get_weather_forecast", + "cosine_score": 0.4608, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "What is the weather forecast and temperature for P\u00e4rnu this weekend?", + "expected": "get_weather_forecast", + "got": "get_weather_forecast", + "cosine_score": 0.4532, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "Show me the 7-day weather forecast for Tallinn", + "expected": "get_weather_forecast", + "got": "get_weather_forecast", + "cosine_score": 0.4621, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "What are the weather conditions including wind speed in Narva today?", + "expected": "get_weather_forecast", + "got": null, + "cosine_score": null, + "rrf_score": null, + "confidence": null, + "pass": false + }, + { + "query": "What is the EUR to USD exchange rate today?", + "expected": "get_exchange_rates", + "got": "get_exchange_rates", + "cosine_score": 0.4068, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "Show me the current currency exchange rates", + "expected": "get_exchange_rates", + "got": "get_exchange_rates", + "cosine_score": 0.5261, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "What is the exchange rate from EUR to Swedish krona?", + "expected": "get_exchange_rates", + "got": null, + "cosine_score": null, + "rrf_score": null, + "confidence": null, + "pass": false + }, + { + "query": "What are the latest forex rates for EUR?", + "expected": "get_exchange_rates", + "got": "get_exchange_rates", + "cosine_score": 0.4201, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "Get country information for Estonia including its capital city", + "expected": "get_country_information", + "got": "get_country_information", + "cosine_score": 0.4763, + "rrf_score": 0.833333, + "confidence": "medium", + "pass": true + }, + { + "query": "What country information is available for Estonia, including official languages?", + "expected": "get_country_information", + "got": "get_country_information", + "cosine_score": 0.4137, + "rrf_score": 0.833333, + "confidence": "medium", + "pass": true + }, + { + "query": "Fetch country details and facts about Estonia", + "expected": "get_country_information", + "got": "get_country_information", + "cosine_score": 0.5485, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "What is the country profile for Estonia?", + "expected": "get_country_information", + "got": null, + "cosine_score": null, + "rrf_score": null, + "confidence": null, + "pass": false + }, + { + "query": "What is the geolocation of IP address 88.196.123.45?", + "expected": "get_ip_geolocation", + "got": null, + "cosine_score": null, + "rrf_score": null, + "confidence": null, + "pass": false + }, + { + "query": "Geolocate this IP address and find which country it belongs to", + "expected": "get_ip_geolocation", + "got": "get_ip_geolocation", + "cosine_score": 0.4792, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "Find the geolocation of an IP address", + "expected": "get_ip_geolocation", + "got": "get_ip_geolocation", + "cosine_score": 0.5684, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "What time is it in Tallinn right now?", + "expected": "get_current_time_by_timezone", + "got": "get_current_time_by_timezone", + "cosine_score": 0.5041, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "What is the current time in the Europe/Tallinn timezone?", + "expected": "get_current_time_by_timezone", + "got": "get_current_time_by_timezone", + "cosine_score": 0.5834, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "What is the current time in Estonia and is it in daylight saving timezone?", + "expected": "get_current_time_by_timezone", + "got": "get_current_time_by_timezone", + "cosine_score": 0.5227, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "What is the air quality in Tallinn today?", + "expected": "get_air_quality", + "got": "get_air_quality", + "cosine_score": 0.5509, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "Show me PM2.5 pollution levels in Tallinn", + "expected": "get_air_quality", + "got": "get_air_quality", + "cosine_score": 0.5537, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "Is the air quality good in Tartu right now?", + "expected": "get_air_quality", + "got": "get_air_quality", + "cosine_score": 0.5289, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "Find the coordinates for Viru 4, Tallinn", + "expected": "get_address_geocoding", + "got": "get_address_geocoding", + "cosine_score": 0.4092, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "Get the geocoding coordinates for Kadriorg Park in Tallinn", + "expected": "get_address_geocoding", + "got": "get_address_geocoding", + "cosine_score": 0.4368, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "What are the GPS coordinates of this address in Estonia?", + "expected": "get_address_geocoding", + "got": "get_address_geocoding", + "cosine_score": 0.4294, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "What is Estonia's GDP this year?", + "expected": "get_gdp_statistics", + "got": "get_gdp_statistics", + "cosine_score": 0.4881, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "What is the economic output of Estonia?", + "expected": "get_gdp_statistics", + "got": "get_gdp_statistics", + "cosine_score": 0.45, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "Show me the GDP growth rate of Estonia over the past 5 years", + "expected": "get_gdp_statistics", + "got": "get_gdp_statistics", + "cosine_score": 0.5406, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "What is the total population of Estonia?", + "expected": "get_population_data", + "got": "get_population_data", + "cosine_score": 0.4216, + "rrf_score": 0.833333, + "confidence": "medium", + "pass": true + }, + { + "query": "What is the total population data for Estonia?", + "expected": "get_population_data", + "got": "get_population_data", + "cosine_score": 0.4915, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "What is the population growth rate of Estonia?", + "expected": "get_population_data", + "got": "get_population_data", + "cosine_score": 0.462, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "Get the word definition for ephemeral", + "expected": "get_word_definition", + "got": "get_word_definition", + "cosine_score": 0.5166, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "Look up the word definition for resilient", + "expected": "get_word_definition", + "got": null, + "cosine_score": null, + "rrf_score": null, + "confidence": null, + "pass": false + }, + { + "query": "Fetch the dictionary definition of the word sustainable", + "expected": "get_word_definition", + "got": "get_word_definition", + "cosine_score": 0.471, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "Where are the bus stops in Tallinn?", + "expected": "get_public_transport_stops", + "got": "get_public_transport_stops", + "cosine_score": 0.5874, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "Show me public transport stops near Tartu city centre", + "expected": "get_public_transport_stops", + "got": "get_public_transport_stops", + "cosine_score": 0.5495, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "What is the average salary in Estonia?", + "expected": "get_average_salary_statistics", + "got": "get_average_salary_statistics", + "cosine_score": 0.5513, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "How much do people earn in Estonia on average?", + "expected": "get_average_salary_statistics", + "got": "get_average_salary_statistics", + "cosine_score": 0.5378, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "What is the average monthly wage in the IT sector in Estonia?", + "expected": "get_average_salary_statistics", + "got": "get_average_salary_statistics", + "cosine_score": 0.5338, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "Look up company registration number 10000000 in Estonia", + "expected": "get_estonian_company_info", + "got": "get_estonian_company_info", + "cosine_score": 0.5831, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "Find details about an Estonian company called Tallinn IT O\u00dc", + "expected": "get_estonian_company_info", + "got": "get_estonian_company_info", + "cosine_score": 0.4899, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "Is this Estonian company still active in the business registry?", + "expected": "get_estonian_company_info", + "got": "get_estonian_company_info", + "cosine_score": 0.5555, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "Reverse geocode the coordinates 59.4370, 24.7536 to get the street address", + "expected": "get_reverse_geocoding", + "got": "get_reverse_geocoding", + "cosine_score": 0.4714, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "Convert GPS coordinates to a street address in Tallinn", + "expected": "get_reverse_geocoding", + "got": "get_reverse_geocoding", + "cosine_score": 0.5075, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "Reverse geocoding for latitude 58.3780 longitude 26.7290 in Tartu", + "expected": "get_reverse_geocoding", + "got": "get_reverse_geocoding", + "cosine_score": 0.471, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "Who is the Prime Minister of Estonia?", + "expected": null, + "got": null, + "cosine_score": null, + "rrf_score": null, + "confidence": null, + "pass": true + }, + { + "query": "What is the best restaurant in Tallinn?", + "expected": null, + "got": null, + "cosine_score": null, + "rrf_score": null, + "confidence": null, + "pass": true + }, + { + "query": "Tell me a random fact about Estonia", + "expected": null, + "got": null, + "cosine_score": null, + "rrf_score": null, + "confidence": null, + "pass": true + }, + { + "query": "What is the meaning of life?", + "expected": null, + "got": null, + "cosine_score": null, + "rrf_score": null, + "confidence": null, + "pass": true + }, + { + "query": "Book me a flight to London", + "expected": null, + "got": null, + "cosine_score": null, + "rrf_score": null, + "confidence": null, + "pass": true + }, + { + "query": "Can you translate this text to Estonian?", + "expected": null, + "got": null, + "cosine_score": null, + "rrf_score": null, + "confidence": null, + "pass": true + }, + { + "query": "What are the visa requirements to visit Estonia?", + "expected": null, + "got": null, + "cosine_score": null, + "rrf_score": null, + "confidence": null, + "pass": true + }, + { + "query": "How do I apply for an Estonian e-Residency?", + "expected": null, + "got": null, + "cosine_score": null, + "rrf_score": null, + "confidence": null, + "pass": true + }, + { + "query": "What is the history of Tallinn Old Town?", + "expected": null, + "got": null, + "cosine_score": null, + "rrf_score": null, + "confidence": null, + "pass": true + }, + { + "query": "Give me a poem about Estonia", + "expected": null, + "got": null, + "cosine_score": null, + "rrf_score": null, + "confidence": null, + "pass": true + } +] \ No newline at end of file From c5582f853046d5e8bfccd25537c63e8de85d1406 Mon Sep 17 00:00:00 2001 From: nuwangeek Date: Wed, 22 Apr 2026 14:45:52 +0530 Subject: [PATCH 5/6] complete semantic searcher evaluation and update to multi point indexing strategy --- .../rag-search/POST/api-tools/search.yml | 81 -- src/api_tool_indexer/constants.py | 20 +- src/api_tool_indexer/main_indexer.py | 191 +++- src/api_tool_indexer/models.py | 20 +- src/api_tool_indexer/qdrant_manager.py | 141 ++- src/llm_orchestration_service_api.py | 83 -- tests/api_tool_eval/batch_index.py | 5 +- tests/api_tool_eval/eval_search.py | 266 +++-- tests/api_tool_eval/results.json | 626 +++++----- tests/api_tool_eval/test-endpoints.json | 150 +++ tests/api_tool_eval/test-results.json | 1010 +++++++++++++++++ 11 files changed, 1854 insertions(+), 739 deletions(-) delete mode 100644 DSL/Ruuter.public/rag-search/POST/api-tools/search.yml create mode 100644 tests/api_tool_eval/test-endpoints.json create mode 100644 tests/api_tool_eval/test-results.json diff --git a/DSL/Ruuter.public/rag-search/POST/api-tools/search.yml b/DSL/Ruuter.public/rag-search/POST/api-tools/search.yml deleted file mode 100644 index 4c19cef2..00000000 --- a/DSL/Ruuter.public/rag-search/POST/api-tools/search.yml +++ /dev/null @@ -1,81 +0,0 @@ -declaration: - call: declare - version: 0.1 - description: "Search API tool endpoints using semantic (hybrid) search against api_tool_collection(test endpoint)" - method: post - accepts: json - returns: json - namespace: rag-search - allowlist: - body: - - field: query - type: string - description: "Natural-language user query to search API endpoints" - - field: top_k - type: integer - description: "Max number of results to return (default: 5)" - - field: environment - type: string - description: "Embedding environment (default: production)" - -extract_request_data: - assign: - query: ${incoming.body.query} - top_k: ${incoming.body.top_k || 5} - environment: ${incoming.body.environment || 'production'} - next: validate_query - -validate_query: - switch: - - condition: "${!query || query.trim() === ''}" - next: return_missing_query - next: execute_search - -return_missing_query: - assign: - error_data: - success: false - error: "MISSING_QUERY" - message: "'query' field is required and must be a non-empty string" - next: return_bad_request - -execute_search: - call: http.post - args: - url: "[#RAG_SEARCH_LLM_SERVICE]/api-tools/search" - body: - query: ${query} - top_k: ${top_k} - environment: ${environment} - result: search_result - on_error: handle_search_error - next: check_search_status - -check_search_status: - switch: - - condition: ${200 <= search_result.response.statusCodeValue && search_result.response.statusCodeValue < 300} - next: return_ok - next: handle_search_error - -handle_search_error: - assign: - error_data: - success: false - error: "SEARCH_FAILED" - message: "Semantic search failed. LLM service may be unavailable." - next: return_server_error - -return_ok: - status: 200 - return: ${search_result.response.body} - next: end - -return_bad_request: - status: 400 - return: ${error_data} - next: end - -return_server_error: - status: 500 - return: ${error_data} - next: end \ No newline at end of file diff --git a/src/api_tool_indexer/constants.py b/src/api_tool_indexer/constants.py index a50b954f..1bdd7b6d 100644 --- a/src/api_tool_indexer/constants.py +++ b/src/api_tool_indexer/constants.py @@ -30,10 +30,19 @@ class ApiToolIndexerConstants: RETRY_DELAY_BASE = 2 # Exponential backoff base (2^attempt seconds) REQUEST_TIMEOUT = 60 # seconds + # Number of example queries generated per endpoint. + # Each example becomes its own Qdrant point so its vector sits in the exact + # language region of the embedding space, enabling short-query matching. + EXAMPLE_QUERY_COUNT = 5 + # Context Enrichment Template - # Mirrors the service workflow (intent_data_enrichment/constants.py). # Full template goes in chunk_prompt; document_prompt is left empty. - # The LLM summarises the chunk content into a rich semantic context. + # + # Multi-point indexing strategy: + # - Each example query line is extracted and stored as its own Qdrant point, + # embedded from that individual sentence alone. + # - The prose + all examples combined become one summary point. + # All in the same language as the endpoint description — no bilingual duplication. CONTEXT_TEMPLATE = """ {full_endpoint_info} @@ -52,18 +61,17 @@ class ApiToolIndexerConstants: - Related concepts and use cases - Common ways users might ask for this functionality in natural language -Then, on a new line, add a section exactly as shown below with 6 to 8 realistic and diverse example questions a real user might ask when they need this endpoint. Cover different phrasings, synonyms, and indirect ways of asking — do not just repeat the description verbatim. +IMPORTANT: Generate the prose context and the example questions in the SAME LANGUAGE as the endpoint description above. However, always use the exact section header "Example queries:" in English regardless of language — this is a required machine-readable marker. IMPORTANT for example queries: This is a system built for Estonian government digital services (Bürokratt). Ground the examples in an Estonian context — use Estonian cities (Tallinn, Tartu, Pärnu, Narva), Estonian institutions, and Estonia-relevant scenarios. Only use non-Estonian locations if the endpoint is explicitly about comparing or fetching data for multiple countries. +Then add a section with exactly {example_count} realistic and diverse example questions a real user might ask when they need this endpoint. Cover different phrasings, synonyms, and indirect ways of asking — do not just repeat the description verbatim. + Example queries: - - - - - -- - -IMPORTANT: Generate everything in the SAME LANGUAGE as the endpoint description above. If the description is in Estonian, respond in Estonian. If in English, respond in English. If in Russian, respond in Russian. Answer only with the enriched context and example queries — nothing else.""" diff --git a/src/api_tool_indexer/main_indexer.py b/src/api_tool_indexer/main_indexer.py index fe25d765..5bddceeb 100644 --- a/src/api_tool_indexer/main_indexer.py +++ b/src/api_tool_indexer/main_indexer.py @@ -2,19 +2,27 @@ Receives raw API EndpointData, enriches it with LLM-generated context, creates hybrid embeddings (dense + sparse), and stores the result in Qdrant -api_tool_collection as a single point per endpoint. +api_tool_collection as multiple points per endpoint. + +Multi-point indexing strategy: + - One 'example' point per example query extracted from the LLM context. + Each query is embedded individually so its vector sits in the correct + language region of the embedding space, enabling accurate short-query matching. + - One 'summary' point containing the combined name + description + enriched context. + This handles broad/paraphrased queries that don't match any single example. Pipeline steps: 1. Build LLM prompt from endpoint name, description, and params 2. Generate context via LLMAPIClient.generate_context() - 3. Build embed text: name + description + context + param descriptions - 4. Create dense embedding via LLMAPIClient.create_embedding() - 5. Create sparse vector via compute_sparse_vector() - 6. Delete existing Qdrant point for idempotent update - 7. Upsert EnrichedEndpoint to api_tool_collection + 3. Parse example query lines from the returned context + 4. Create dense + sparse embeddings per example query (example points) + 5. Create dense + sparse embedding for combined summary text (summary point) + 6. Delete all existing Qdrant points for this endpoint (filter-based, idempotent) + 7. Upsert all points to api_tool_collection 8. Return IndexingResult """ +import re import sys import json import asyncio @@ -111,10 +119,13 @@ async def _generate_context_for_endpoint( logger.info(f"params_summary : {params_summary}") + # Escape braces in the URL to prevent str.format() from treating path + # parameter templates like {id} as format placeholders (KeyError). + safe_url = endpoint_data.url.replace("{", "{{").replace("}", "}}") full_endpoint_info = ( f"Endpoint: {endpoint_data.name}\n" f"Method: {endpoint_data.method}\n" - f"URL: {endpoint_data.url}\n" + f"URL: {safe_url}\n" f"Description: {endpoint_data.description}\n" f"Parameters: {params_summary}" ) @@ -124,6 +135,7 @@ async def _generate_context_for_endpoint( name=endpoint_data.name, description=endpoint_data.description, params_summary=params_summary, + example_count=ApiToolIndexerConstants.EXAMPLE_QUERY_COUNT, ) logger.debug( @@ -197,9 +209,54 @@ async def _generate_context_for_endpoint( raise RuntimeError(error_msg) +_EXAMPLE_SECTION_HEADER = re.compile(r"^example queries\s*:", re.IGNORECASE) + + +def _parse_example_queries(context: str) -> List[str]: + """Extract example query lines from the LLM-generated context. + + Scans for the 'Example queries:' section header and collects every + subsequent '- ' line until the section ends. + + Args: + context: Raw LLM-generated context string from generate_context(). + + Returns: + List of example query strings, deduplicated and preserving order. + """ + examples: List[str] = [] + in_section = False + + for line in context.splitlines(): + stripped = line.strip() + if _EXAMPLE_SECTION_HEADER.match(stripped): + in_section = True + continue + if in_section: + if stripped.startswith("- "): + examples.append(stripped[2:].strip()) + elif stripped and not stripped.startswith("#"): + # Non-empty, non-comment line that isn't a list item ends the section + in_section = False + + # Deduplicate preserving order + seen: set[str] = set() + unique: List[str] = [] + for ex in examples: + if ex and ex not in seen: + seen.add(ex) + unique.append(ex) + return unique + + async def index_endpoint(endpoint_data: EndpointData) -> IndexingResult: """Index one API endpoint into Qdrant api_tool_collection. + Creates multiple points per endpoint: + - One 'example' point per parsed example query, embedded from that + individual sentence so the vector sits in the correct language region. + - One 'summary' point embedded from the combined name + description + context. + Args: endpoint_data: Raw endpoint data from mock_endpoints table. @@ -213,7 +270,6 @@ async def index_endpoint(endpoint_data: EndpointData) -> IndexingResult: ) try: - # Steps 1–5: LLM enrichment and embedding async with LLMAPIClient( api_base_url=ApiToolIndexerConstants.DEFAULT_API_BASE_URL, environment=ApiToolIndexerConstants.DEFAULT_ENVIRONMENT, @@ -222,84 +278,123 @@ async def index_endpoint(endpoint_data: EndpointData) -> IndexingResult: retry_delay_base=ApiToolIndexerConstants.RETRY_DELAY_BASE, timeout=ApiToolIndexerConstants.REQUEST_TIMEOUT, ) as api_client: - # Step 1-2: Generate LLM enriched context - logger.info("Step 1/5: Generating LLM enriched context") + # Step 1: Generate LLM enriched context (prose + example queries) + logger.info("Step 1/4: Generating LLM enriched context") enriched_context = await _generate_context_for_endpoint( api_client, endpoint_data ) - # Step 3: Build embed text combining all semantic signal + # Step 2: Parse example query lines from the context + example_queries = _parse_example_queries(enriched_context) + if not example_queries: + logger.warning( + f"No example queries parsed from context for endpoint '{endpoint_id}'. " + "The LLM output may not contain an 'Example queries:' section. " + "Only a summary point will be indexed — search accuracy may be reduced." + ) + else: + logger.info( + f"Step 2/4: Parsed {len(example_queries)} example queries from context" + ) + + # Step 3: Embed each example query individually → example points + logger.info( + f"Step 3/4: Creating embeddings for {len(example_queries)} example points" + ) + enriched_points: List[EnrichedEndpoint] = [] + + for i, example in enumerate(example_queries): + logger.debug( + f" Embedding example {i + 1}/{len(example_queries)}: " + f"'{example[:80]}{'...' if len(example) > 80 else ''}'" + ) + ex_embedding = await api_client.create_embedding(example) + ex_sparse = compute_sparse_vector(example) + enriched_points.append( + EnrichedEndpoint( + endpoint_id=endpoint_id, + name=endpoint_data.name, + description=endpoint_data.description, + url=endpoint_data.url, + method=endpoint_data.method, + params=endpoint_data.params, + enriched_context=enriched_context, + service_id=endpoint_data.service_id, + point_type="example", + example_text=example, + embedding=ex_embedding, + sparse_indices=ex_sparse.indices, + sparse_values=ex_sparse.values, + ) + ) + + # Step 4: Embed combined summary text → summary point + logger.info("Step 4/4: Creating summary point embedding") params_summary = _build_params_summary(endpoint_data.params) - embed_text = ( + summary_text = ( f"{endpoint_data.name}. " f"{endpoint_data.description}. " f"{enriched_context}. " f"Parameters: {params_summary}" ) + summary_embedding = await api_client.create_embedding(summary_text) - # Step 4: Create dense embedding vector - logger.info("Step 2/5: Creating dense embedding vector") - dense_embedding = await api_client.create_embedding(embed_text) - - # Step 5: Create sparse (BM25) vector - synchronous, after closing HTTP session - logger.info("Step 3/5: Computing sparse (BM25) vector") - sparse_vec = compute_sparse_vector(embed_text) - - # Build EnrichedEndpoint ready for Qdrant storage - enriched = EnrichedEndpoint( - endpoint_id=endpoint_id, - name=endpoint_data.name, - description=endpoint_data.description, - url=endpoint_data.url, - method=endpoint_data.method, - params=endpoint_data.params, - enriched_context=enriched_context, - service_id=endpoint_data.service_id, - embedding=dense_embedding, - sparse_indices=sparse_vec.indices, - sparse_values=sparse_vec.values, + # Sparse vectors are CPU-bound — computed after the HTTP session closes + summary_sparse = compute_sparse_vector(summary_text) + enriched_points.append( + EnrichedEndpoint( + endpoint_id=endpoint_id, + name=endpoint_data.name, + description=endpoint_data.description, + url=endpoint_data.url, + method=endpoint_data.method, + params=endpoint_data.params, + enriched_context=enriched_context, + service_id=endpoint_data.service_id, + point_type="summary", + embedding=summary_embedding, + sparse_indices=summary_sparse.indices, + sparse_values=summary_sparse.values, + ) ) - # Steps 6-7: Qdrant operations (separate try/finally ensures connection is closed) + # Qdrant operations — separate block so the connection is always closed qdrant = ApiToolQdrantManager() try: qdrant.connect() qdrant.ensure_collection() - # Step 6: Delete existing point for idempotent update - logger.info("Step 4/5: Deleting existing Qdrant point (idempotent update)") - deleted = qdrant.delete_endpoint_point(endpoint_id) + # Delete all existing points for this endpoint (filter-based, idempotent) + deleted = qdrant.delete_endpoint_points(endpoint_id) if not deleted: logger.error( - f"Failed to delete existing Qdrant point for endpoint '{endpoint_id}'. " + f"Failed to delete existing points for endpoint '{endpoint_id}'. " "Aborting upsert to prevent stale data." ) return IndexingResult( success=False, endpoint_id=endpoint_id, message="Qdrant delete failed before upsert", - error="delete_endpoint_point returned False", + error="delete_endpoint_points returned False", ) - # Step 7: Upsert the enriched endpoint - logger.info("Step 5/5: Upserting endpoint into api_tool_collection") - upserted = qdrant.upsert_endpoint(enriched) - + upserted = qdrant.upsert_endpoint_points(enriched_points) finally: qdrant.close() - # Step 8: Return result + n_examples = len(example_queries) if upserted: logger.success( - f"Endpoint '{endpoint_id}' (name='{endpoint_data.name}') " - "indexed successfully" + f"Endpoint '{endpoint_id}' (name='{endpoint_data.name}') indexed successfully " + f"({n_examples} example points + 1 summary point)" ) return IndexingResult( success=True, endpoint_id=endpoint_id, message=( f"Endpoint '{endpoint_data.name}' indexed successfully into " - f"api_tool_collection (dim={len(dense_embedding)})" + f"api_tool_collection " + f"({n_examples} example points + 1 summary point)" ), ) else: @@ -307,7 +402,7 @@ async def index_endpoint(endpoint_data: EndpointData) -> IndexingResult: success=False, endpoint_id=endpoint_id, message="Qdrant upsert failed", - error="upsert_endpoint returned False", + error="upsert_endpoint_points returned False", ) except Exception as e: diff --git a/src/api_tool_indexer/models.py b/src/api_tool_indexer/models.py index 94d75d4d..6333d2fd 100644 --- a/src/api_tool_indexer/models.py +++ b/src/api_tool_indexer/models.py @@ -44,8 +44,14 @@ class EndpointData(BaseModel): class EnrichedEndpoint(BaseModel): """Enriched endpoint data ready for storage in Qdrant api_tool_collection. - One point per endpoint is stored. - The payload stored in Qdrant includes all fields needed by the agentic loop + Multiple points are stored per endpoint: + - One 'example' point per example query — embedded from that sentence alone, + so the vector sits in the correct language region of the embedding space. + - One 'summary' point — embedded from name + description + full enriched context. + + All points share the same endpoint_id payload field so the searcher can + deduplicate results back to a single endpoint after retrieval. + The payload on every point contains all fields needed by the agentic loop so no additional DB roundtrip is needed after a semantic match. """ @@ -63,6 +69,16 @@ class EnrichedEndpoint(BaseModel): ) service_id: Optional[str] = Field(default=None, description="Parent service UUID") + # Point type — controls which text was embedded for this point + point_type: str = Field( + default="summary", + description="'example' (individual query text) or 'summary' (full context)", + ) + example_text: Optional[str] = Field( + default=None, + description="The example query string for 'example' points; None for 'summary'", + ) + # Vector fields (populated by indexer pipeline) embedding: List[float] = Field( default_factory=list, description="Dense embedding vector (3072-dim)" diff --git a/src/api_tool_indexer/qdrant_manager.py b/src/api_tool_indexer/qdrant_manager.py index cbbf4a92..de2fa0e8 100644 --- a/src/api_tool_indexer/qdrant_manager.py +++ b/src/api_tool_indexer/qdrant_manager.py @@ -2,17 +2,21 @@ for the api_tool_collection used by the API Tool Calling workflow. """ -from typing import Any, Dict, Optional +import uuid +from typing import Any, Dict, List, Optional from loguru import logger from qdrant_client import QdrantClient from qdrant_client.models import ( Distance, - VectorParams, + FieldCondition, + Filter, + FilterSelector, + MatchValue, PointStruct, - SparseVectorParams, SparseIndexParams, SparseVector, - PointIdsList, + SparseVectorParams, + VectorParams, ) from api_tool_indexer.constants import ApiToolIndexerConstants @@ -25,7 +29,10 @@ class ApiToolQdrantManager: """Manages Qdrant operations for api_tool_collection with hybrid search. - One point per endpoint is stored. + Multiple points are stored per endpoint: + - One 'example' point per example query + - One 'summary' point for the full combined context + All points share the same endpoint_id payload field for deduplication. """ def __init__( @@ -162,89 +169,125 @@ def _create_collection(self) -> None: ) logger.success(f"Collection '{self.collection_name}' created successfully") - def delete_endpoint_point(self, endpoint_id: str) -> bool: - """Delete the Qdrant point for a given endpoint. + def delete_endpoint_points(self, endpoint_id: str) -> bool: + """Delete all Qdrant points for a given endpoint. - Used before re-indexing to ensure idempotent updates, and when - an endpoint is deleted from the mock_endpoints table. + Uses a payload filter on 'endpoint_id' to remove all example and summary + points belonging to this endpoint. Called before re-indexing to ensure + idempotent updates, and when an endpoint is removed from the DB. Args: - endpoint_id: UUID of the endpoint to delete + endpoint_id: UUID of the endpoint whose points should be deleted. Returns: - True if successful, False otherwise + True if successful, False otherwise. """ try: if not self.client: raise RuntimeError(_CLIENT_NOT_INITIALIZED) - logger.info( - f"Deleting existing point for endpoint '{endpoint_id}' from Qdrant" - ) + logger.info(f"Deleting all points for endpoint '{endpoint_id}' from Qdrant") self.client.delete( collection_name=self.collection_name, - points_selector=PointIdsList(points=[endpoint_id]), + points_selector=FilterSelector( + filter=Filter( + must=[ + FieldCondition( + key="endpoint_id", + match=MatchValue(value=endpoint_id), + ) + ] + ) + ), + ) + logger.success( + f"Successfully deleted all points for endpoint '{endpoint_id}'" ) - logger.success(f"Successfully deleted point for endpoint '{endpoint_id}'") return True except Exception as e: - logger.error(f"Failed to delete point for endpoint '{endpoint_id}': {e}") + logger.error(f"Failed to delete points for endpoint '{endpoint_id}': {e}") return False - def upsert_endpoint(self, enriched: EnrichedEndpoint) -> bool: - """Upsert one enriched endpoint point to Qdrant. + def upsert_endpoint_points(self, enriched_points: List[EnrichedEndpoint]) -> bool: + """Upsert multiple enriched endpoint points to Qdrant. + + Each point gets a deterministic UUID derived from endpoint_id + index so + upserts are idempotent. All points carry the full endpoint payload so no + additional DB roundtrip is needed after a semantic match. + + Payload fields stored on every point: + endpoint_id, name, description, url, method, params, + enriched_context, service_id, point_type, example_text (example only) Args: - enriched: EnrichedEndpoint with dense/sparse vectors populated + enriched_points: List of EnrichedEndpoint instances (examples + summary). Returns: - True if successful, False otherwise + True if all points upserted successfully, False otherwise. """ try: if not self.client: raise RuntimeError(_CLIENT_NOT_INITIALIZED) - logger.info(f"Upserting point for endpoint '{enriched.endpoint_id}'") - - payload = { - "endpoint_id": enriched.endpoint_id, - "name": enriched.name, - "description": enriched.description, - "url": enriched.url, - "method": enriched.method, - "params": enriched.params, - "enriched_context": enriched.enriched_context, - "service_id": enriched.service_id, - } - - vectors: Dict[str, Any] = { - ApiToolIndexerConstants.DENSE_VECTOR_NAME: enriched.embedding, - } - if enriched.sparse_indices: - vectors[ApiToolIndexerConstants.SPARSE_VECTOR_NAME] = SparseVector( - indices=enriched.sparse_indices, - values=enriched.sparse_values, - ) + if not enriched_points: + logger.warning("No points to upsert") + return True - point = PointStruct( - id=enriched.endpoint_id, # use endpoint UUID directly as point ID - vector=vectors, - payload=payload, + endpoint_id = enriched_points[0].endpoint_id + logger.info( + f"Upserting {len(enriched_points)} points for endpoint '{endpoint_id}'" ) + points: List[PointStruct] = [] + for idx, enriched in enumerate(enriched_points): + # Deterministic UUID: same input always produces the same point ID + point_id = str( + uuid.uuid5(uuid.NAMESPACE_DNS, f"{enriched.endpoint_id}_{idx}") + ) + + payload: Dict[str, Any] = { + "endpoint_id": enriched.endpoint_id, + "name": enriched.name, + "description": enriched.description, + "url": enriched.url, + "method": enriched.method, + "params": enriched.params, + "enriched_context": enriched.enriched_context, + "service_id": enriched.service_id, + "point_type": enriched.point_type, + } + if enriched.example_text is not None: + payload["example_text"] = enriched.example_text + + vectors: Dict[str, Any] = { + ApiToolIndexerConstants.DENSE_VECTOR_NAME: enriched.embedding, + } + if enriched.sparse_indices: + vectors[ApiToolIndexerConstants.SPARSE_VECTOR_NAME] = SparseVector( + indices=enriched.sparse_indices, + values=enriched.sparse_values, + ) + + points.append(PointStruct(id=point_id, vector=vectors, payload=payload)) + self.client.upsert( collection_name=self.collection_name, - points=[point], + points=points, ) + + n_examples = sum(1 for p in enriched_points if p.point_type == "example") + n_summary = sum(1 for p in enriched_points if p.point_type == "summary") logger.success( - f"Successfully upserted point for endpoint '{enriched.endpoint_id}'" + f"Successfully upserted {len(points)} points for endpoint '{endpoint_id}' " + f"({n_examples} example + {n_summary} summary)" ) return True except Exception as e: logger.error( - f"Failed to upsert point for endpoint '{enriched.endpoint_id}': {e}" + f"Failed to upsert points for endpoint " + f"'{enriched_points[0].endpoint_id if enriched_points else '?'}': {e}" ) return False diff --git a/src/llm_orchestration_service_api.py b/src/llm_orchestration_service_api.py index e8dddb7e..ddd66a9a 100644 --- a/src/llm_orchestration_service_api.py +++ b/src/llm_orchestration_service_api.py @@ -257,89 +257,6 @@ async def health_check(request: Request) -> dict[str, str]: } -@app.post( - "/api-tools/search", - status_code=status.HTTP_200_OK, - summary="[TEST] Search API tool endpoints by natural-language query", - description=( - "Test-only endpoint for evaluating semantic retrieval accuracy against " - "api_tool_collection. Bypasses classifier and all other workflows. " - "Returns ranked endpoints with cosine scores and confidence levels." - ), -) -async def api_tools_search( - http_request: Request, - body: Dict[str, Any], -) -> Dict[str, Any]: - """Run hybrid semantic search against api_tool_collection. - - Use this endpoint from Postman to evaluate whether the correct API endpoint - is returned for a given natural-language query. - - Request body: - query (str): Natural-language user query. Required. - top_k (int): Max results to return. Default: 5. - environment (str): Embedding environment. Default: "production". - - Response fields per result: - endpoint_id: UUID of the matched endpoint - name: Endpoint function name - description: Human-readable description - method: HTTP method (GET / POST) - url: Actual API URL - params: List of parameter schemas - cosine_score: How similar the query is to this endpoint (0.0 - 1.0) - confidence: "high" / "medium" (see threshold constants) - """ - from tool_classifier.api_semantic_searcher import APISemanticSearcher - - query = body.get("query", "").strip() - if not query: - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="'query' field is required and must be a non-empty string", - ) - - top_k = int(body.get("top_k", 5)) - environment = body.get("environment", "production") - - orchestration_service = getattr( - http_request.app.state, "orchestration_service", None - ) - if orchestration_service is None: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Orchestration service not initialized", - ) - - try: - searcher = APISemanticSearcher(embedding_service=orchestration_service) - results = await searcher.search( - query=query, - environment=environment, - top_k=top_k, - ) - await searcher.aclose() - - return { - "query": query, - "total_results": len(results), - "results": [r.to_dict() for r in results], - "interpretation": { - "high_confidence": "Endpoint can be used directly — query is a very clear match", - "medium_confidence": "Possible match — may need LLM disambiguation in production", - "no_results": "No endpoint matched above the minimum threshold (0.45)", - }, - } - - except Exception as e: - logger.error(f"API tools search failed: {e}", exc_info=True) - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Search failed: {str(e)}", - ) - - @app.post( "/orchestrate", response_model=OrchestrationResponse, diff --git a/tests/api_tool_eval/batch_index.py b/tests/api_tool_eval/batch_index.py index bff78958..0aa0555a 100644 --- a/tests/api_tool_eval/batch_index.py +++ b/tests/api_tool_eval/batch_index.py @@ -1,10 +1,9 @@ """ -Batch Indexer — sends all endpoints from endpoints.json to POST /api-tools/index. +Batch Indexer — sends all endpoints from test-endpoints.json to POST /api-tools/index. Usage: python batch_index.py python batch_index.py --ruuter-url http://localhost:8086 - python batch_index.py --skip-existing # skip endpoints already in Qdrant """ import argparse @@ -14,7 +13,7 @@ import requests -ENDPOINTS_FILE = Path(__file__).parent / "endpoints.json" +ENDPOINTS_FILE = Path(__file__).parent / "test-endpoints.json" DEFAULT_RUUTER_URL = "http://localhost:8086" INDEX_ENDPOINT = "/rag-search/api-tools/index" diff --git a/tests/api_tool_eval/eval_search.py b/tests/api_tool_eval/eval_search.py index 4fcf5eff..a37590e6 100644 --- a/tests/api_tool_eval/eval_search.py +++ b/tests/api_tool_eval/eval_search.py @@ -25,168 +25,211 @@ SEARCH_ENDPOINT = "/rag-search/api-tools/search" # ============================================================================ -# Evaluation Dataset +# Evaluation Dataset — aligned with test-endpoints.json (15 endpoints) # Format: (query, expected_endpoint_name or None for "no match expected") # ============================================================================ EVAL_QUERIES = [ - # --- get_national_holidays --- - ("What are the national holidays in Estonia?", "get_national_holidays"), + # --- get_public_holidays --- + ("What are the public holidays in Estonia this year?", "get_public_holidays"), + ("List official public holidays in Estonia for 2025", "get_public_holidays"), + ("When are the national public holidays in Estonia?", "get_public_holidays"), ( - "What are the upcoming national holidays in Estonia this year?", - "get_national_holidays", + "Show me all public days off in Estonia between January and June", + "get_public_holidays", ), - ("List all public days off in Estonia this year", "get_national_holidays"), - ("Show me national holidays for Estonia", "get_national_holidays"), - ("What are the official Estonian public holidays?", "get_national_holidays"), + ("What are the official non-working days in Estonia?", "get_public_holidays"), + # Estonian + ("Millised on Eesti riigipühad sel aastal?", "get_public_holidays"), + ("Millal on Eestis ametlikud riigipühad 2025. aastal?", "get_public_holidays"), + ("Näita mulle Eesti riigipühi jaanuarist juunini", "get_public_holidays"), # --- get_school_holidays --- ("When are the school holidays in Estonia?", "get_school_holidays"), - ("What are the school term breaks in Estonia?", "get_school_holidays"), - ("When does school summer break start in Estonia?", "get_school_holidays"), - # --- get_current_electricity_price --- + ("What are the school term breaks in Estonia this year?", "get_school_holidays"), + ("When does school summer break start in Estonia in 2025?", "get_school_holidays"), ( - "What is the current electricity price in Estonia?", - "get_current_electricity_price", + "Show me school holiday periods in Estonia for spring 2025", + "get_school_holidays", ), + # Estonian + ("Millal on Eesti koolide koolivaheajad?", "get_school_holidays"), + ("Millal algab koolide suvepuhkus Eestis 2025. aastal?", "get_school_holidays"), + ("Näita mulle kevadise koolivaheaja aegu Eestis", "get_school_holidays"), + # --- get_electricity_prices --- + ("What are the electricity market prices in Estonia?", "get_electricity_prices"), ( - "How much does electricity cost right now in Estonia?", - "get_current_electricity_price", + "Show me electricity prices for the past week in Estonia", + "get_electricity_prices", ), ( - "Show me the real-time energy market price in Estonia", - "get_current_electricity_price", + "Fetch energy market prices between January and March 2025", + "get_electricity_prices", ), ( - "What is the spot price for electricity in Estonia today?", - "get_current_electricity_price", + "What was the electricity spot price in Estonia last month?", + "get_electricity_prices", ), - # --- get_electricity_price_history --- + # Estonian + ("Millised on elektrituruhinnad Eestis?", "get_electricity_prices"), + ("Näita elektrihindu eelmise nädala kohta Eestis", "get_electricity_prices"), + ("Mis oli elektrihind Eestis eelmisel kuul?", "get_electricity_prices"), + # --- get_vehicle_tax_info --- + ("Calculate vehicle tax for registration number 123ABC", "get_vehicle_tax_info"), ( - "Show me the electricity price history for Estonia over the last month", - "get_electricity_price_history", + "How much is the vehicle tax for my car with plate 456XYZ?", + "get_vehicle_tax_info", ), + ("What is the car tax based on my registration number?", "get_vehicle_tax_info"), + # Estonian + ("Arvuta sõidukimaks registreerimisnumbri 123ABC alusel", "get_vehicle_tax_info"), + ("Kui suur on minu auto maks numbrimärgi 456XYZ järgi?", "get_vehicle_tax_info"), ( - "Show me historical electricity prices for Estonia in January 2024", - "get_electricity_price_history", + "Mis on mootorsõidukimaks minu auto registreerimisnumbri alusel?", + "get_vehicle_tax_info", ), + # --- get_parliament_votings --- ( - "Fetch the electricity price history for Estonia for the past 30 days", - "get_electricity_price_history", + "Show me the latest parliament voting records in Estonia", + "get_parliament_votings", ), - # --- get_unemployment_rate --- - ("What is the unemployment rate in Estonia?", "get_unemployment_rate"), - ("How many people are unemployed in Estonia this year?", "get_unemployment_rate"), - ("Show me the latest jobless statistics for Estonia", "get_unemployment_rate"), - ("What percentage of Estonians are unemployed?", "get_unemployment_rate"), - # --- get_weather_forecast --- - ("What is the weather forecast for Tallinn tomorrow?", "get_weather_forecast"), - ("What is the weather forecast for Tartu next week?", "get_weather_forecast"), + ("What did the Riigikogu vote on recently?", "get_parliament_votings"), ( - "What is the weather forecast and temperature for Pärnu this weekend?", - "get_weather_forecast", + "Retrieve parliamentary voting decisions from the Estonian parliament", + "get_parliament_votings", ), - ("Show me the 7-day weather forecast for Tallinn", "get_weather_forecast"), + ("What laws were voted on in the Estonian parliament?", "get_parliament_votings"), + # Estonian + ("Näita Riigikogu viimaseid hääletusprotokolle", "get_parliament_votings"), + ("Mille üle hääletas Riigikogu hiljuti?", "get_parliament_votings"), + ("Milliseid seadusi hääletati Eesti parlamendis?", "get_parliament_votings"), + # --- get_parliament_participation_stats --- ( - "What are the weather conditions including wind speed in Narva today?", - "get_weather_forecast", + "How often do Estonian parliament members attend sessions?", + "get_parliament_participation_stats", ), - # --- get_exchange_rates --- - ("What is the EUR to USD exchange rate today?", "get_exchange_rates"), - ("Show me the current currency exchange rates", "get_exchange_rates"), - ("What is the exchange rate from EUR to Swedish krona?", "get_exchange_rates"), - ("What are the latest forex rates for EUR?", "get_exchange_rates"), - # --- get_country_information --- ( - "Get country information for Estonia including its capital city", - "get_country_information", + "Show me parliament member attendance statistics", + "get_parliament_participation_stats", ), ( - "What country information is available for Estonia, including official languages?", - "get_country_information", + "Which MPs have the best attendance record in the Riigikogu?", + "get_parliament_participation_stats", ), - ("Fetch country details and facts about Estonia", "get_country_information"), - ("What is the country profile for Estonia?", "get_country_information"), - # --- get_ip_geolocation --- - ("What is the geolocation of IP address 88.196.123.45?", "get_ip_geolocation"), + # Estonian ( - "Geolocate this IP address and find which country it belongs to", - "get_ip_geolocation", + "Kui tihti osalevad Riigikogu liikmed istungitel?", + "get_parliament_participation_stats", ), - ("Find the geolocation of an IP address", "get_ip_geolocation"), - # --- get_current_time_by_timezone --- - ("What time is it in Tallinn right now?", "get_current_time_by_timezone"), ( - "What is the current time in the Europe/Tallinn timezone?", - "get_current_time_by_timezone", + "Näita Riigikogu liikmete kohaloleku statistikat", + "get_parliament_participation_stats", ), ( - "What is the current time in Estonia and is it in daylight saving timezone?", - "get_current_time_by_timezone", + "Millistel saadikutel on Riigikogu parim kohalolekurekord?", + "get_parliament_participation_stats", ), - # --- get_air_quality --- - ("What is the air quality in Tallinn today?", "get_air_quality"), - ("Show me PM2.5 pollution levels in Tallinn", "get_air_quality"), - ("Is the air quality good in Tartu right now?", "get_air_quality"), - # --- get_address_geocoding --- - ("Find the coordinates for Viru 4, Tallinn", "get_address_geocoding"), + # --- get_initiatives --- + ("Show me a list of active citizen initiatives in Estonia", "get_initiatives"), + ("What public initiatives are currently available?", "get_initiatives"), + ("List all citizen initiatives on rahvaalgatus.ee", "get_initiatives"), + # Estonian + ("Näita mulle aktiivsete kodanike algatuste nimekirja Eestis", "get_initiatives"), + ("Millised rahvaalgatused on praegu saadaval?", "get_initiatives"), + ("Loetle kõik algatused rahvaalgatus.ee lehel", "get_initiatives"), + # --- get_initiative_details --- + ("Get details about citizen initiative with ID abc123", "get_initiative_details"), ( - "Get the geocoding coordinates for Kadriorg Park in Tallinn", - "get_address_geocoding", + "Show me more information about a specific public initiative", + "get_initiative_details", ), + ("Fetch the details of initiative ID xyz789", "get_initiative_details"), + # Estonian + ("Too andmed kodanike algatuse ID abc123 kohta", "get_initiative_details"), + ("Näita mulle üksikasju konkreetse rahvaalgatuse kohta", "get_initiative_details"), + ("Too algatuse ID xyz789 üksikasjad", "get_initiative_details"), + # --- get_initiative_events --- ( - "What are the GPS coordinates of this address in Estonia?", - "get_address_geocoding", + "What are the latest events related to citizen initiatives?", + "get_initiative_events", ), - # --- get_gdp_statistics --- - ("What is Estonia's GDP this year?", "get_gdp_statistics"), - ("What is the economic output of Estonia?", "get_gdp_statistics"), + ("Show me updates and events for public initiatives", "get_initiative_events"), ( - "Show me the GDP growth rate of Estonia over the past 5 years", - "get_gdp_statistics", + "Are there any new events for citizen initiatives in Estonia?", + "get_initiative_events", ), - # --- get_population_data --- - ("What is the total population of Estonia?", "get_population_data"), - ("What is the total population data for Estonia?", "get_population_data"), - ("What is the population growth rate of Estonia?", "get_population_data"), - # --- get_word_definition --- - ("Get the word definition for ephemeral", "get_word_definition"), - ("Look up the word definition for resilient", "get_word_definition"), - ("Fetch the dictionary definition of the word sustainable", "get_word_definition"), - # --- get_public_transport_stops --- - ("Where are the bus stops in Tallinn?", "get_public_transport_stops"), + # Estonian ( - "Show me public transport stops near Tartu city centre", - "get_public_transport_stops", + "Millised on viimased kodanike algatustega seotud sündmused?", + "get_initiative_events", ), - # --- get_average_salary_statistics --- - ("What is the average salary in Estonia?", "get_average_salary_statistics"), - ("How much do people earn in Estonia on average?", "get_average_salary_statistics"), + ("Näita rahvaalgatuste uuendusi ja sündmusi", "get_initiative_events"), + ("Kas Eestis on uusi sündmusi kodanike algatuste kohta?", "get_initiative_events"), + # --- search_address --- + ("Search for the address Viru 4 in Tallinn", "search_address"), + ("Find the location of Kadriorg Park in Tallinn", "search_address"), + ("Look up an address or place name in Estonia", "search_address"), + ("Search for a street address in Tartu", "search_address"), + # Estonian + ("Otsi aadressi Viru 4 Tallinnas", "search_address"), + ("Leia Kadrioru pargi asukoht Tallinnas", "search_address"), + ("Otsi tänavaaadress Tartus", "search_address"), + # --- get_population_statistics --- + ("What is the population of Estonia?", "get_population_statistics"), + ("Show me population statistics data for Estonia", "get_population_statistics"), + ("Fetch demographic statistics for Estonia", "get_population_statistics"), ( - "What is the average monthly wage in the IT sector in Estonia?", - "get_average_salary_statistics", + "What is the population breakdown by age group in Estonia?", + "get_population_statistics", ), - # --- get_estonian_company_info --- + # Estonian + ("Milline on Eesti rahvaarv?", "get_population_statistics"), + ("Näita mulle Eesti rahvastikustatistika andmeid", "get_population_statistics"), ( - "Look up company registration number 10000000 in Estonia", - "get_estonian_company_info", + "Milline on Eesti rahvastiku jaotus vanuserühmade kaupa?", + "get_population_statistics", ), + # --- get_economic_statistics --- + ("Show me economic statistics for Estonia", "get_economic_statistics"), + ("What is the GDP and economic output of Estonia?", "get_economic_statistics"), ( - "Find details about an Estonian company called Tallinn IT OÜ", - "get_estonian_company_info", + "Fetch economic data for Estonia from the statistics office", + "get_economic_statistics", ), + # Estonian + ("Näita mulle Eesti majandusstatistikat", "get_economic_statistics"), + ("Mis on Eesti SKP ja majanduslik toodang?", "get_economic_statistics"), + ("Too majandusandmed Eesti statistikaametist", "get_economic_statistics"), + # --- get_labor_statistics --- + ("What is the unemployment rate in Estonia?", "get_labor_statistics"), + ("Show me labor and employment statistics for Estonia", "get_labor_statistics"), + ("How many people are employed in Estonia?", "get_labor_statistics"), + ("Fetch workforce and jobless statistics for Estonia", "get_labor_statistics"), + # Estonian + ("Milline on töötuse määr Eestis?", "get_labor_statistics"), + ("Näita mulle Eesti tööjõu ja tööhõive statistikat", "get_labor_statistics"), + ("Kui palju inimesi töötab Eestis?", "get_labor_statistics"), + # --- get_current_weather --- + ("What is the current weather in Tallinn?", "get_current_weather"), + ("Show me the current weather conditions in Estonia", "get_current_weather"), ( - "Is this Estonian company still active in the business registry?", - "get_estonian_company_info", + "What is the temperature right now at the Tallinn weather station?", + "get_current_weather", ), - # --- get_reverse_geocoding --- - ( - "Reverse geocode the coordinates 59.4370, 24.7536 to get the street address", - "get_reverse_geocoding", - ), - ("Convert GPS coordinates to a street address in Tallinn", "get_reverse_geocoding"), + # Estonian + ("Milline on praegune ilm Tallinnas?", "get_current_weather"), + ("Näita mulle praeguseid ilmastikuolusid Eestis", "get_current_weather"), + ("Mis on praegune temperatuur Tallinna ilmajaamas?", "get_current_weather"), + # --- get_weather_forecast --- + ("What is the weather forecast for Tallinn tomorrow?", "get_weather_forecast"), + ("Show me the upcoming weather forecast for Tartu", "get_weather_forecast"), + ("What will the weather be like in Estonia next week?", "get_weather_forecast"), ( - "Reverse geocoding for latitude 58.3780 longitude 26.7290 in Tartu", - "get_reverse_geocoding", + "Give me a weather forecast for the next few days in Estonia", + "get_weather_forecast", ), + # Estonian + ("Milline on ilmaprognoos Tallinnas homme?", "get_weather_forecast"), + ("Näita mulle Tartu eelseisvat ilmaprognoosi", "get_weather_forecast"), + ("Milline on ilm Eestis järgmisel nädalal?", "get_weather_forecast"), # --- NEGATIVE queries — should return NO matching results --- ("Who is the Prime Minister of Estonia?", None), ("What is the best restaurant in Tallinn?", None), @@ -198,6 +241,10 @@ ("How do I apply for an Estonian e-Residency?", None), ("What is the history of Tallinn Old Town?", None), ("Give me a poem about Estonia", None), + # Estonian negatives + ("Kes on Eesti peaminister?", None), + ("Mis on parim restoran Tallinnas?", None), + ("Mis on elu mõte?", None), ] @@ -232,6 +279,7 @@ def evaluate(ruuter_url: str, delay: float = 0.5) -> list: "expected": expected, "got": "ERROR", "cosine_score": None, + "rrf_score": None, "confidence": None, "pass": False, "error": "Request failed", diff --git a/tests/api_tool_eval/results.json b/tests/api_tool_eval/results.json index 1c089e58..7a8e8715 100644 --- a/tests/api_tool_eval/results.json +++ b/tests/api_tool_eval/results.json @@ -1,286 +1,223 @@ [ { - "query": "What are the national holidays in Estonia?", - "expected": "get_national_holidays", - "got": "get_national_holidays", - "cosine_score": 0.4781, - "rrf_score": 1.0, - "confidence": "medium", + "query": "What are the public holidays in Estonia this year?", + "expected": "get_public_holidays", + "got": "get_public_holidays", + "cosine_score": 0.6562, + "rrf_score": 0.666667, + "confidence": "high", "pass": true }, { - "query": "What are the upcoming national holidays in Estonia this year?", - "expected": "get_national_holidays", - "got": "get_national_holidays", - "cosine_score": 0.4738, - "rrf_score": 1.0, - "confidence": "medium", + "query": "List official public holidays in Estonia for 2025", + "expected": "get_public_holidays", + "got": "get_public_holidays", + "cosine_score": 0.7149, + "rrf_score": 0.7, + "confidence": "high", "pass": true }, { - "query": "List all public days off in Estonia this year", - "expected": "get_national_holidays", - "got": "get_national_holidays", - "cosine_score": 0.4398, - "rrf_score": 0.833333, - "confidence": "medium", + "query": "When are the national public holidays in Estonia?", + "expected": "get_public_holidays", + "got": "get_public_holidays", + "cosine_score": 0.631, + "rrf_score": 0.7, + "confidence": "high", "pass": true }, { - "query": "Show me national holidays for Estonia", - "expected": "get_national_holidays", - "got": "get_national_holidays", - "cosine_score": 0.5081, - "rrf_score": 1.0, + "query": "Show me all public days off in Estonia between January and June", + "expected": "get_public_holidays", + "got": "get_public_holidays", + "cosine_score": 0.5639, + "rrf_score": 0.75, "confidence": "medium", "pass": true }, { - "query": "What are the official Estonian public holidays?", - "expected": "get_national_holidays", - "got": "get_national_holidays", - "cosine_score": 0.4171, - "rrf_score": 1.0, - "confidence": "medium", + "query": "What are the official non-working days in Estonia?", + "expected": "get_public_holidays", + "got": "get_public_holidays", + "cosine_score": 0.6714, + "rrf_score": 0.666667, + "confidence": "high", "pass": true }, { "query": "When are the school holidays in Estonia?", "expected": "get_school_holidays", "got": "get_school_holidays", - "cosine_score": 0.4986, - "rrf_score": 1.0, - "confidence": "medium", + "cosine_score": 0.7116, + "rrf_score": 0.75, + "confidence": "high", "pass": true }, { - "query": "What are the school term breaks in Estonia?", + "query": "What are the school term breaks in Estonia this year?", "expected": "get_school_holidays", "got": "get_school_holidays", - "cosine_score": 0.5082, - "rrf_score": 1.0, - "confidence": "medium", + "cosine_score": 0.706, + "rrf_score": 0.7, + "confidence": "high", "pass": true }, { - "query": "When does school summer break start in Estonia?", + "query": "When does school summer break start in Estonia in 2025?", "expected": "get_school_holidays", "got": "get_school_holidays", - "cosine_score": 0.4147, - "rrf_score": 1.0, + "cosine_score": 0.5987, + "rrf_score": 0.642857, "confidence": "medium", "pass": true }, { - "query": "What is the current electricity price in Estonia?", - "expected": "get_current_electricity_price", - "got": "get_current_electricity_price", - "cosine_score": 0.7182, - "rrf_score": 1.0, - "confidence": "high", - "pass": true - }, - { - "query": "How much does electricity cost right now in Estonia?", - "expected": "get_current_electricity_price", - "got": "get_current_electricity_price", - "cosine_score": 0.6776, - "rrf_score": 1.0, - "confidence": "high", + "query": "Show me school holiday periods in Estonia for spring 2025", + "expected": "get_school_holidays", + "got": "get_school_holidays", + "cosine_score": 0.591, + "rrf_score": 0.7, + "confidence": "medium", "pass": true }, { - "query": "Show me the real-time energy market price in Estonia", - "expected": "get_current_electricity_price", - "got": "get_current_electricity_price", - "cosine_score": 0.7339, - "rrf_score": 1.0, + "query": "What are the electricity market prices in Estonia?", + "expected": "get_electricity_prices", + "got": "get_electricity_prices", + "cosine_score": 0.6814, + "rrf_score": 0.833333, "confidence": "high", "pass": true }, { - "query": "What is the spot price for electricity in Estonia today?", - "expected": "get_current_electricity_price", - "got": "get_current_electricity_price", - "cosine_score": 0.6782, - "rrf_score": 1.0, + "query": "Show me electricity prices for the past week in Estonia", + "expected": "get_electricity_prices", + "got": "get_electricity_prices", + "cosine_score": 0.6186, + "rrf_score": 0.666667, "confidence": "high", "pass": true }, { - "query": "Show me the electricity price history for Estonia over the last month", - "expected": "get_electricity_price_history", - "got": "get_electricity_price_history", - "cosine_score": 0.6211, - "rrf_score": 0.833333, - "confidence": "medium", - "pass": true - }, - { - "query": "Show me historical electricity prices for Estonia in January 2024", - "expected": "get_electricity_price_history", - "got": "get_electricity_price_history", - "cosine_score": 0.606, - "rrf_score": 0.833333, - "confidence": "medium", - "pass": true - }, - { - "query": "Fetch the electricity price history for Estonia for the past 30 days", - "expected": "get_electricity_price_history", - "got": "get_electricity_price_history", - "cosine_score": 0.7049, + "query": "Fetch energy market prices between January and March 2025", + "expected": "get_electricity_prices", + "got": "get_electricity_prices", + "cosine_score": 0.5003, "rrf_score": 1.0, "confidence": "medium", "pass": true }, { - "query": "What is the unemployment rate in Estonia?", - "expected": "get_unemployment_rate", - "got": "get_unemployment_rate", - "cosine_score": 0.6034, - "rrf_score": 1.0, + "query": "What was the electricity spot price in Estonia last month?", + "expected": "get_electricity_prices", + "got": "get_electricity_prices", + "cosine_score": 0.641, + "rrf_score": 0.642857, "confidence": "high", "pass": true }, { - "query": "How many people are unemployed in Estonia this year?", - "expected": "get_unemployment_rate", - "got": "get_unemployment_rate", - "cosine_score": 0.5537, - "rrf_score": 0.5, - "confidence": "medium", - "pass": true - }, - { - "query": "Show me the latest jobless statistics for Estonia", - "expected": "get_unemployment_rate", - "got": "get_unemployment_rate", - "cosine_score": 0.5786, + "query": "Calculate vehicle tax for registration number 123ABC", + "expected": "get_vehicle_tax_info", + "got": "get_vehicle_tax_info", + "cosine_score": 0.6402, "rrf_score": 0.833333, - "confidence": "medium", + "confidence": "high", "pass": true }, { - "query": "What percentage of Estonians are unemployed?", - "expected": "get_unemployment_rate", - "got": "get_unemployment_rate", - "cosine_score": 0.5649, + "query": "How much is the vehicle tax for my car with plate 456XYZ?", + "expected": "get_vehicle_tax_info", + "got": "get_vehicle_tax_info", + "cosine_score": 0.4854, "rrf_score": 0.5, "confidence": "medium", "pass": true }, { - "query": "What is the weather forecast for Tallinn tomorrow?", - "expected": "get_weather_forecast", - "got": "get_weather_forecast", - "cosine_score": 0.4984, - "rrf_score": 1.0, + "query": "What is the car tax based on my registration number?", + "expected": "get_vehicle_tax_info", + "got": "get_vehicle_tax_info", + "cosine_score": 0.5449, + "rrf_score": 0.7, "confidence": "medium", "pass": true }, { - "query": "What is the weather forecast for Tartu next week?", - "expected": "get_weather_forecast", - "got": "get_weather_forecast", - "cosine_score": 0.4608, - "rrf_score": 1.0, - "confidence": "medium", - "pass": true - }, - { - "query": "What is the weather forecast and temperature for P\u00e4rnu this weekend?", - "expected": "get_weather_forecast", - "got": "get_weather_forecast", - "cosine_score": 0.4532, - "rrf_score": 1.0, + "query": "Show me the latest parliament voting records in Estonia", + "expected": "get_parliament_votings", + "got": "get_parliament_votings", + "cosine_score": 0.5909, + "rrf_score": 0.5, "confidence": "medium", "pass": true }, { - "query": "Show me the 7-day weather forecast for Tallinn", - "expected": "get_weather_forecast", - "got": "get_weather_forecast", - "cosine_score": 0.4621, - "rrf_score": 1.0, + "query": "What did the Riigikogu vote on recently?", + "expected": "get_parliament_votings", + "got": "get_parliament_votings", + "cosine_score": 0.5407, + "rrf_score": 0.5, "confidence": "medium", "pass": true }, { - "query": "What are the weather conditions including wind speed in Narva today?", - "expected": "get_weather_forecast", - "got": null, - "cosine_score": null, - "rrf_score": null, - "confidence": null, - "pass": false - }, - { - "query": "What is the EUR to USD exchange rate today?", - "expected": "get_exchange_rates", - "got": "get_exchange_rates", - "cosine_score": 0.4068, - "rrf_score": 1.0, + "query": "Retrieve parliamentary voting decisions from the Estonian parliament", + "expected": "get_parliament_votings", + "got": "get_parliament_votings", + "cosine_score": 0.7032, + "rrf_score": 0.5, "confidence": "medium", "pass": true }, { - "query": "Show me the current currency exchange rates", - "expected": "get_exchange_rates", - "got": "get_exchange_rates", - "cosine_score": 0.5261, - "rrf_score": 1.0, + "query": "What laws were voted on in the Estonian parliament?", + "expected": "get_parliament_votings", + "got": "get_parliament_votings", + "cosine_score": 0.4812, + "rrf_score": 0.5, "confidence": "medium", "pass": true }, { - "query": "What is the exchange rate from EUR to Swedish krona?", - "expected": "get_exchange_rates", - "got": null, - "cosine_score": null, - "rrf_score": null, - "confidence": null, - "pass": false - }, - { - "query": "What are the latest forex rates for EUR?", - "expected": "get_exchange_rates", - "got": "get_exchange_rates", - "cosine_score": 0.4201, - "rrf_score": 1.0, + "query": "How often do Estonian parliament members attend sessions?", + "expected": "get_parliament_participation_stats", + "got": "get_parliament_participation_stats", + "cosine_score": 0.559, + "rrf_score": 0.5, "confidence": "medium", "pass": true }, { - "query": "Get country information for Estonia including its capital city", - "expected": "get_country_information", - "got": "get_country_information", - "cosine_score": 0.4763, + "query": "Show me parliament member attendance statistics", + "expected": "get_parliament_participation_stats", + "got": "get_parliament_participation_stats", + "cosine_score": 0.5782, "rrf_score": 0.833333, "confidence": "medium", "pass": true }, { - "query": "What country information is available for Estonia, including official languages?", - "expected": "get_country_information", - "got": "get_country_information", - "cosine_score": 0.4137, - "rrf_score": 0.833333, - "confidence": "medium", + "query": "Which MPs have the best attendance record in the Riigikogu?", + "expected": "get_parliament_participation_stats", + "got": "get_parliament_participation_stats", + "cosine_score": 0.7401, + "rrf_score": 0.5, + "confidence": "high", "pass": true }, { - "query": "Fetch country details and facts about Estonia", - "expected": "get_country_information", - "got": "get_country_information", - "cosine_score": 0.5485, - "rrf_score": 1.0, - "confidence": "medium", + "query": "Show me a list of active citizen initiatives in Estonia", + "expected": "get_initiatives", + "got": "get_initiatives", + "cosine_score": 0.6301, + "rrf_score": 0.5, + "confidence": "high", "pass": true }, { - "query": "What is the country profile for Estonia?", - "expected": "get_country_information", + "query": "What public initiatives are currently available?", + "expected": "get_initiatives", "got": null, "cosine_score": null, "rrf_score": null, @@ -288,179 +225,152 @@ "pass": false }, { - "query": "What is the geolocation of IP address 88.196.123.45?", - "expected": "get_ip_geolocation", - "got": null, - "cosine_score": null, - "rrf_score": null, - "confidence": null, - "pass": false - }, - { - "query": "Geolocate this IP address and find which country it belongs to", - "expected": "get_ip_geolocation", - "got": "get_ip_geolocation", - "cosine_score": 0.4792, - "rrf_score": 1.0, - "confidence": "medium", - "pass": true - }, - { - "query": "Find the geolocation of an IP address", - "expected": "get_ip_geolocation", - "got": "get_ip_geolocation", - "cosine_score": 0.5684, - "rrf_score": 1.0, - "confidence": "medium", + "query": "List all citizen initiatives on rahvaalgatus.ee", + "expected": "get_initiatives", + "got": "get_initiatives", + "cosine_score": 0.6211, + "rrf_score": 0.833333, + "confidence": "high", "pass": true }, { - "query": "What time is it in Tallinn right now?", - "expected": "get_current_time_by_timezone", - "got": "get_current_time_by_timezone", - "cosine_score": 0.5041, + "query": "Get details about citizen initiative with ID abc123", + "expected": "get_initiative_details", + "got": "get_initiative_details", + "cosine_score": 0.6135, "rrf_score": 1.0, - "confidence": "medium", + "confidence": "high", "pass": true }, { - "query": "What is the current time in the Europe/Tallinn timezone?", - "expected": "get_current_time_by_timezone", - "got": "get_current_time_by_timezone", - "cosine_score": 0.5834, - "rrf_score": 1.0, + "query": "Show me more information about a specific public initiative", + "expected": "get_initiative_details", + "got": "get_initiative_details", + "cosine_score": 0.5031, + "rrf_score": 0.75, "confidence": "medium", "pass": true }, { - "query": "What is the current time in Estonia and is it in daylight saving timezone?", - "expected": "get_current_time_by_timezone", - "got": "get_current_time_by_timezone", - "cosine_score": 0.5227, + "query": "Fetch the details of initiative ID xyz789", + "expected": "get_initiative_details", + "got": "get_initiative_details", + "cosine_score": 0.6288, "rrf_score": 1.0, - "confidence": "medium", + "confidence": "high", "pass": true }, { - "query": "What is the air quality in Tallinn today?", - "expected": "get_air_quality", - "got": "get_air_quality", - "cosine_score": 0.5509, - "rrf_score": 1.0, + "query": "What are the latest events related to citizen initiatives?", + "expected": "get_initiative_events", + "got": "get_initiative_events", + "cosine_score": 0.5195, + "rrf_score": 0.333333, "confidence": "medium", "pass": true }, { - "query": "Show me PM2.5 pollution levels in Tallinn", - "expected": "get_air_quality", - "got": "get_air_quality", - "cosine_score": 0.5537, - "rrf_score": 1.0, + "query": "Show me updates and events for public initiatives", + "expected": "get_initiative_events", + "got": "get_initiative_events", + "cosine_score": 0.5383, + "rrf_score": 0.642857, "confidence": "medium", "pass": true }, { - "query": "Is the air quality good in Tartu right now?", - "expected": "get_air_quality", - "got": "get_air_quality", - "cosine_score": 0.5289, - "rrf_score": 1.0, - "confidence": "medium", - "pass": true + "query": "Are there any new events for citizen initiatives in Estonia?", + "expected": "get_initiative_events", + "got": null, + "cosine_score": null, + "rrf_score": null, + "confidence": null, + "pass": false }, { - "query": "Find the coordinates for Viru 4, Tallinn", - "expected": "get_address_geocoding", - "got": "get_address_geocoding", - "cosine_score": 0.4092, - "rrf_score": 1.0, + "query": "Search for the address Viru 4 in Tallinn", + "expected": "search_address", + "got": "search_address", + "cosine_score": 0.5366, + "rrf_score": 0.5, "confidence": "medium", "pass": true }, { - "query": "Get the geocoding coordinates for Kadriorg Park in Tallinn", - "expected": "get_address_geocoding", - "got": "get_address_geocoding", - "cosine_score": 0.4368, - "rrf_score": 1.0, + "query": "Find the location of Kadriorg Park in Tallinn", + "expected": "search_address", + "got": "search_address", + "cosine_score": 0.4156, + "rrf_score": 0.5, "confidence": "medium", "pass": true }, { - "query": "What are the GPS coordinates of this address in Estonia?", - "expected": "get_address_geocoding", - "got": "get_address_geocoding", - "cosine_score": 0.4294, - "rrf_score": 1.0, + "query": "Look up an address or place name in Estonia", + "expected": "search_address", + "got": "search_address", + "cosine_score": 0.5359, + "rrf_score": 0.642857, "confidence": "medium", "pass": true }, { - "query": "What is Estonia's GDP this year?", - "expected": "get_gdp_statistics", - "got": "get_gdp_statistics", - "cosine_score": 0.4881, - "rrf_score": 1.0, - "confidence": "medium", + "query": "Search for a street address in Tartu", + "expected": "search_address", + "got": "search_address", + "cosine_score": 0.609, + "rrf_score": 0.642857, + "confidence": "high", "pass": true }, { - "query": "What is the economic output of Estonia?", - "expected": "get_gdp_statistics", - "got": "get_gdp_statistics", - "cosine_score": 0.45, + "query": "What is the population of Estonia?", + "expected": "get_population_statistics", + "got": "get_population_statistics", + "cosine_score": 0.4635, "rrf_score": 1.0, "confidence": "medium", "pass": true }, { - "query": "Show me the GDP growth rate of Estonia over the past 5 years", - "expected": "get_gdp_statistics", - "got": "get_gdp_statistics", - "cosine_score": 0.5406, + "query": "Show me population statistics data for Estonia", + "expected": "get_population_statistics", + "got": "get_population_statistics", + "cosine_score": 0.5567, "rrf_score": 1.0, "confidence": "medium", "pass": true }, { - "query": "What is the total population of Estonia?", - "expected": "get_population_data", - "got": "get_population_data", - "cosine_score": 0.4216, + "query": "Fetch demographic statistics for Estonia", + "expected": "get_population_statistics", + "got": "get_population_statistics", + "cosine_score": 0.6124, "rrf_score": 0.833333, "confidence": "medium", "pass": true }, { - "query": "What is the total population data for Estonia?", - "expected": "get_population_data", - "got": "get_population_data", - "cosine_score": 0.4915, + "query": "What is the population breakdown by age group in Estonia?", + "expected": "get_population_statistics", + "got": "get_population_statistics", + "cosine_score": 0.4676, "rrf_score": 1.0, "confidence": "medium", "pass": true }, { - "query": "What is the population growth rate of Estonia?", - "expected": "get_population_data", - "got": "get_population_data", - "cosine_score": 0.462, - "rrf_score": 1.0, - "confidence": "medium", - "pass": true - }, - { - "query": "Get the word definition for ephemeral", - "expected": "get_word_definition", - "got": "get_word_definition", - "cosine_score": 0.5166, - "rrf_score": 1.0, + "query": "Show me economic statistics for Estonia", + "expected": "get_economic_statistics", + "got": "get_economic_statistics", + "cosine_score": 0.5485, + "rrf_score": 0.833333, "confidence": "medium", "pass": true }, { - "query": "Look up the word definition for resilient", - "expected": "get_word_definition", + "query": "What is the GDP and economic output of Estonia?", + "expected": "get_economic_statistics", "got": null, "cosine_score": null, "rrf_score": null, @@ -468,110 +378,110 @@ "pass": false }, { - "query": "Fetch the dictionary definition of the word sustainable", - "expected": "get_word_definition", - "got": "get_word_definition", - "cosine_score": 0.471, + "query": "Fetch economic data for Estonia from the statistics office", + "expected": "get_economic_statistics", + "got": "get_economic_statistics", + "cosine_score": 0.6043, "rrf_score": 1.0, - "confidence": "medium", + "confidence": "high", "pass": true }, { - "query": "Where are the bus stops in Tallinn?", - "expected": "get_public_transport_stops", - "got": "get_public_transport_stops", - "cosine_score": 0.5874, - "rrf_score": 1.0, + "query": "What is the unemployment rate in Estonia?", + "expected": "get_labor_statistics", + "got": "get_labor_statistics", + "cosine_score": 0.5585, + "rrf_score": 0.5, "confidence": "medium", "pass": true }, { - "query": "Show me public transport stops near Tartu city centre", - "expected": "get_public_transport_stops", - "got": "get_public_transport_stops", - "cosine_score": 0.5495, - "rrf_score": 1.0, + "query": "Show me labor and employment statistics for Estonia", + "expected": "get_labor_statistics", + "got": "get_labor_statistics", + "cosine_score": 0.5584, + "rrf_score": 0.5, "confidence": "medium", "pass": true }, { - "query": "What is the average salary in Estonia?", - "expected": "get_average_salary_statistics", - "got": "get_average_salary_statistics", - "cosine_score": 0.5513, - "rrf_score": 1.0, + "query": "How many people are employed in Estonia?", + "expected": "get_labor_statistics", + "got": "get_labor_statistics", + "cosine_score": 0.5152, + "rrf_score": 0.5, "confidence": "medium", "pass": true }, { - "query": "How much do people earn in Estonia on average?", - "expected": "get_average_salary_statistics", - "got": "get_average_salary_statistics", - "cosine_score": 0.5378, - "rrf_score": 1.0, + "query": "Fetch workforce and jobless statistics for Estonia", + "expected": "get_labor_statistics", + "got": "get_labor_statistics", + "cosine_score": 0.5649, + "rrf_score": 0.5, "confidence": "medium", "pass": true }, { - "query": "What is the average monthly wage in the IT sector in Estonia?", - "expected": "get_average_salary_statistics", - "got": "get_average_salary_statistics", - "cosine_score": 0.5338, - "rrf_score": 1.0, - "confidence": "medium", + "query": "What is the current weather in Tallinn?", + "expected": "get_current_weather", + "got": "get_current_weather", + "cosine_score": 0.7829, + "rrf_score": 0.5, + "confidence": "high", "pass": true }, { - "query": "Look up company registration number 10000000 in Estonia", - "expected": "get_estonian_company_info", - "got": "get_estonian_company_info", - "cosine_score": 0.5831, - "rrf_score": 1.0, - "confidence": "medium", + "query": "Show me the current weather conditions in Estonia", + "expected": "get_current_weather", + "got": "get_current_weather", + "cosine_score": 0.6238, + "rrf_score": 0.5, + "confidence": "high", "pass": true }, { - "query": "Find details about an Estonian company called Tallinn IT O\u00dc", - "expected": "get_estonian_company_info", - "got": "get_estonian_company_info", - "cosine_score": 0.4899, - "rrf_score": 1.0, + "query": "What is the temperature right now at the Tallinn weather station?", + "expected": "get_current_weather", + "got": "get_current_weather", + "cosine_score": 0.7113, + "rrf_score": 0.5, "confidence": "medium", "pass": true }, { - "query": "Is this Estonian company still active in the business registry?", - "expected": "get_estonian_company_info", - "got": "get_estonian_company_info", - "cosine_score": 0.5555, - "rrf_score": 1.0, - "confidence": "medium", + "query": "What is the weather forecast for Tallinn tomorrow?", + "expected": "get_weather_forecast", + "got": "get_weather_forecast", + "cosine_score": 0.7875, + "rrf_score": 0.5, + "confidence": "high", "pass": true }, { - "query": "Reverse geocode the coordinates 59.4370, 24.7536 to get the street address", - "expected": "get_reverse_geocoding", - "got": "get_reverse_geocoding", - "cosine_score": 0.4714, - "rrf_score": 1.0, - "confidence": "medium", - "pass": true + "query": "Show me the upcoming weather forecast for Tartu", + "expected": "get_weather_forecast", + "got": "get_current_weather", + "cosine_score": 0.7177, + "rrf_score": 0.5, + "confidence": "high", + "pass": false }, { - "query": "Convert GPS coordinates to a street address in Tallinn", - "expected": "get_reverse_geocoding", - "got": "get_reverse_geocoding", - "cosine_score": 0.5075, - "rrf_score": 1.0, + "query": "What will the weather be like in Estonia next week?", + "expected": "get_weather_forecast", + "got": "get_weather_forecast", + "cosine_score": 0.5939, + "rrf_score": 0.5, "confidence": "medium", "pass": true }, { - "query": "Reverse geocoding for latitude 58.3780 longitude 26.7290 in Tartu", - "expected": "get_reverse_geocoding", - "got": "get_reverse_geocoding", - "cosine_score": 0.471, - "rrf_score": 1.0, + "query": "Give me a weather forecast for the next few days in Estonia", + "expected": "get_weather_forecast", + "got": "get_weather_forecast", + "cosine_score": 0.5743, + "rrf_score": 0.5, "confidence": "medium", "pass": true }, diff --git a/tests/api_tool_eval/test-endpoints.json b/tests/api_tool_eval/test-endpoints.json new file mode 100644 index 00000000..07d35266 --- /dev/null +++ b/tests/api_tool_eval/test-endpoints.json @@ -0,0 +1,150 @@ +[ + { + "endpointId": "1f9c2a11-3c6e-4a91-8b77-0b2e7c1a1001", + "name": "get_public_holidays", + "description": "Too riiklikud pühad konkreetse riigi kohta etteantud ajavahemikus.", + "url": "https://openholidaysapi.org/PublicHolidays", + "method": "GET", + "params": [ + { "name": "countryIsoCode", "type": "string", "required": true, "description": "Kahetäheline ISO riigikood (nt EE, LV)" }, + { "name": "validFrom", "type": "date", "required": true, "description": "Alguskuupäev (YYYY-MM-DD)" }, + { "name": "validTo", "type": "date", "required": true, "description": "Lõppkuupäev (YYYY-MM-DD)" }, + { "name": "languageIsoCode", "type": "string", "required": false, "description": "Valikuline keelekood (nt ET, EN)" } + ] + }, + { + "endpointId": "1f9c2a11-3c6e-4a91-8b77-0b2e7c1a1002", + "name": "get_school_holidays", + "description": "Too koolivaheaegade andmed konkreetse riigi kohta etteantud ajavahemikus.", + "url": "https://openholidaysapi.org/SchoolHolidays", + "method": "GET", + "params": [ + { "name": "countryIsoCode", "type": "string", "required": true, "description": "Kahetäheline ISO riigikood" }, + { "name": "validFrom", "type": "date", "required": true, "description": "Alguskuupäev (YYYY-MM-DD)" }, + { "name": "validTo", "type": "date", "required": true, "description": "Lõppkuupäev (YYYY-MM-DD)" } + ] + }, + { + "endpointId": "2a7b3c22-9d5f-4b1e-9c44-1d3a9e220003", + "name": "get_electricity_prices", + "description": "Too elektrituruhinnad etteantud ajavahemiku kohta.", + "url": "https://dashboard.elering.ee/api/nps/price", + "method": "GET", + "params": [ + { "name": "start", "type": "datetime", "required": true, "description": "Algusaeg (ISO 8601 formaat)" }, + { "name": "end", "type": "datetime", "required": true, "description": "Lõppaeg (ISO 8601 formaat)" } + ] + }, + { + "endpointId": "3b8d4e33-7f6a-4c2d-a911-2c4b8f330004", + "name": "get_vehicle_tax_info", + "description": "Arvuta sõidukimaks registreerimisnumbri alusel.", + "url": "https://avalik.emta.ee/msm-public/v1/vehicle-tax", + "method": "POST", + "params": [ + { "name": "registrationNumber", "type": "string", "required": true, "description": "Sõiduki registreerimisnumber" } + ] + }, + { + "endpointId": "4c9e5f44-1a2b-4d3e-b822-3d5c9f440005", + "name": "get_parliament_votings", + "description": "Too Riigikogu hääletusprotokolid ja otsused.", + "url": "https://api.riigikogu.ee/api/votings", + "method": "GET", + "params": [] + }, + { + "endpointId": "4c9e5f44-1a2b-4d3e-b822-3d5c9f440006", + "name": "get_parliament_participation_stats", + "description": "Too Riigikogu liikmete kohaloleku ja osalemise statistika.", + "url": "https://api.riigikogu.ee/api/statistics/participation", + "method": "GET", + "params": [] + }, + { + "endpointId": "5d0f6a55-2b3c-4e4f-c933-4e6d0a550007", + "name": "get_initiatives", + "description": "Too kodanike rahvaalgatuste nimekiri.", + "url": "https://rahvaalgatus.ee/initiatives", + "method": "GET", + "params": [ + { "name": "page", "type": "number", "required": false, "description": "Lehekülje number lehekülgede kaupa" } + ] + }, + { + "endpointId": "5d0f6a55-2b3c-4e4f-c933-4e6d0a550008", + "name": "get_initiative_details", + "description": "Too üksikasjalikku teavet konkreetse kodanike rahvaalgatuse kohta. Algatuse ID lisatakse tee osana alusURL-ile (nt /initiatives/123).", + "url": "https://rahvaalgatus.ee/initiatives", + "method": "GET", + "params": [ + { "name": "id", "type": "string", "required": true, "description": "Algatuse kordumatu identifikaator — lisatakse tee osana URL-ile" } + ] + }, + { + "endpointId": "5d0f6a55-2b3c-4e4f-c933-4e6d0a550009", + "name": "get_initiative_events", + "description": "Too kodanike rahvaalgatustega seotud sündmused ja uuendused.", + "url": "https://rahvaalgatus.ee/initiative-events", + "method": "GET", + "params": [] + }, + { + "endpointId": "6e1a7b66-3c4d-5f5a-d044-5f7e1b660010", + "name": "search_address", + "description": "Otsi aadresse või asukohti märksõnapäringu abil.", + "url": "https://inaadress.maaamet.ee/inaadress/gazetteer", + "method": "GET", + "params": [ + { "name": "address", "type": "string", "required": true, "description": "Aadressi või kohanime otsingupäring" } + ] + }, + { + "endpointId": "7f2b8c77-4d5e-6a6b-e155-6a8f2c770011", + "name": "get_population_statistics", + "description": "Too rahvastikustatistika andmed struktureeritud päringu abil.", + "url": "https://andmed.stat.ee/api/v1/en/stat/IA021", + "method": "POST", + "params": [ + { "name": "query", "type": "object", "required": true, "description": "JSON-päringu keha andmestiku filtrite ja mõõtmetega" } + ] + }, + { + "endpointId": "7f2b8c77-4d5e-6a6b-e155-6a8f2c770012", + "name": "get_economic_statistics", + "description": "Too majandusstatistika andmed struktureeritud päringu abil.", + "url": "https://andmed.stat.ee/api/v1/en/stat/LE27", + "method": "POST", + "params": [ + { "name": "query", "type": "object", "required": true, "description": "JSON-päringu keha andmestiku filtrite ja mõõtmetega" } + ] + }, + { + "endpointId": "7f2b8c77-4d5e-6a6b-e155-6a8f2c770013", + "name": "get_labor_statistics", + "description": "Too tööjõu ja tööhõive statistika andmed struktureeritud päringu abil.", + "url": "https://andmed.stat.ee/api/v1/en/stat/TT330", + "method": "POST", + "params": [ + { "name": "query", "type": "object", "required": true, "description": "JSON-päringu keha andmestiku filtrite ja mõõtmetega" } + ] + }, + { + "endpointId": "8a3c9d88-5e6f-7b7c-f266-7b9a3d880014", + "name": "get_current_weather", + "description": "Too praegused ja kombineeritud ilmaandmed.", + "url": "https://publicapi.envir.ee/v1/combinedWeatherData", + "method": "GET", + "params": [ + { "name": "station", "type": "string", "required": false, "description": "Ilmajaama identifikaator" } + ] + }, + { + "endpointId": "8a3c9d88-5e6f-7b7c-f266-7b9a3d880015", + "name": "get_weather_forecast", + "description": "Too ilmaprognoos tulevaste perioodide kohta.", + "url": "https://ilmmicroservice.envir.ee/api/forecasts", + "method": "GET", + "params": [] + } +] \ No newline at end of file diff --git a/tests/api_tool_eval/test-results.json b/tests/api_tool_eval/test-results.json new file mode 100644 index 00000000..6965e2cc --- /dev/null +++ b/tests/api_tool_eval/test-results.json @@ -0,0 +1,1010 @@ +[ + { + "query": "What are the public holidays in Estonia this year?", + "expected": "get_public_holidays", + "got": "get_public_holidays", + "cosine_score": 0.6777, + "rrf_score": 0.5, + "confidence": "medium", + "pass": true + }, + { + "query": "List official public holidays in Estonia for 2025", + "expected": "get_public_holidays", + "got": "get_public_holidays", + "cosine_score": 0.6234, + "rrf_score": 0.5, + "confidence": "high", + "pass": true + }, + { + "query": "When are the national public holidays in Estonia?", + "expected": "get_public_holidays", + "got": "get_public_holidays", + "cosine_score": 0.5978, + "rrf_score": 0.5, + "confidence": "medium", + "pass": true + }, + { + "query": "Show me all public days off in Estonia between January and June", + "expected": "get_public_holidays", + "got": "get_public_holidays", + "cosine_score": 0.5606, + "rrf_score": 0.5, + "confidence": "medium", + "pass": true + }, + { + "query": "What are the official non-working days in Estonia?", + "expected": "get_public_holidays", + "got": "get_public_holidays", + "cosine_score": 0.6518, + "rrf_score": 0.5, + "confidence": "high", + "pass": true + }, + { + "query": "Millised on Eesti riigip\u00fchad sel aastal?", + "expected": "get_public_holidays", + "got": "get_public_holidays", + "cosine_score": 0.6896, + "rrf_score": 0.5, + "confidence": "high", + "pass": true + }, + { + "query": "Millal on Eestis ametlikud riigip\u00fchad 2025. aastal?", + "expected": "get_public_holidays", + "got": "get_public_holidays", + "cosine_score": 0.6969, + "rrf_score": 0.5, + "confidence": "high", + "pass": true + }, + { + "query": "N\u00e4ita mulle Eesti riigip\u00fchi jaanuarist juunini", + "expected": "get_public_holidays", + "got": "get_public_holidays", + "cosine_score": 0.5552, + "rrf_score": 0.5, + "confidence": "medium", + "pass": true + }, + { + "query": "When are the school holidays in Estonia?", + "expected": "get_school_holidays", + "got": "get_school_holidays", + "cosine_score": 0.6782, + "rrf_score": 0.5, + "confidence": "high", + "pass": true + }, + { + "query": "What are the school term breaks in Estonia this year?", + "expected": "get_school_holidays", + "got": "get_school_holidays", + "cosine_score": 0.646, + "rrf_score": 0.5, + "confidence": "medium", + "pass": true + }, + { + "query": "When does school summer break start in Estonia in 2025?", + "expected": "get_school_holidays", + "got": "get_school_holidays", + "cosine_score": 0.6939, + "rrf_score": 0.833333, + "confidence": "high", + "pass": true + }, + { + "query": "Show me school holiday periods in Estonia for spring 2025", + "expected": "get_school_holidays", + "got": "get_school_holidays", + "cosine_score": 0.5997, + "rrf_score": 0.666667, + "confidence": "medium", + "pass": true + }, + { + "query": "Millal on Eesti koolide koolivaheajad?", + "expected": "get_school_holidays", + "got": "get_school_holidays", + "cosine_score": 0.7407, + "rrf_score": 0.642857, + "confidence": "high", + "pass": true + }, + { + "query": "Millal algab koolide suvepuhkus Eestis 2025. aastal?", + "expected": "get_school_holidays", + "got": "get_school_holidays", + "cosine_score": 0.6872, + "rrf_score": 0.5, + "confidence": "high", + "pass": true + }, + { + "query": "N\u00e4ita mulle kevadise koolivaheaja aegu Eestis", + "expected": "get_school_holidays", + "got": "get_school_holidays", + "cosine_score": 0.6195, + "rrf_score": 0.5, + "confidence": "high", + "pass": true + }, + { + "query": "What are the electricity market prices in Estonia?", + "expected": "get_electricity_prices", + "got": "get_electricity_prices", + "cosine_score": 0.6332, + "rrf_score": 0.5, + "confidence": "high", + "pass": true + }, + { + "query": "Show me electricity prices for the past week in Estonia", + "expected": "get_electricity_prices", + "got": "get_electricity_prices", + "cosine_score": 0.6837, + "rrf_score": 0.5, + "confidence": "high", + "pass": true + }, + { + "query": "Fetch energy market prices between January and March 2025", + "expected": "get_electricity_prices", + "got": "get_electricity_prices", + "cosine_score": 0.432, + "rrf_score": 0.5, + "confidence": "medium", + "pass": true + }, + { + "query": "What was the electricity spot price in Estonia last month?", + "expected": "get_electricity_prices", + "got": "get_electricity_prices", + "cosine_score": 0.6716, + "rrf_score": 0.5, + "confidence": "high", + "pass": true + }, + { + "query": "Millised on elektrituruhinnad Eestis?", + "expected": "get_electricity_prices", + "got": "get_electricity_prices", + "cosine_score": 0.6218, + "rrf_score": 0.5, + "confidence": "medium", + "pass": true + }, + { + "query": "N\u00e4ita elektrihindu eelmise n\u00e4dala kohta Eestis", + "expected": "get_electricity_prices", + "got": "get_electricity_prices", + "cosine_score": 0.7159, + "rrf_score": 0.5, + "confidence": "medium", + "pass": true + }, + { + "query": "Mis oli elektrihind Eestis eelmisel kuul?", + "expected": "get_electricity_prices", + "got": "get_electricity_prices", + "cosine_score": 0.6676, + "rrf_score": 0.642857, + "confidence": "high", + "pass": true + }, + { + "query": "Calculate vehicle tax for registration number 123ABC", + "expected": "get_vehicle_tax_info", + "got": "get_vehicle_tax_info", + "cosine_score": 0.653, + "rrf_score": 0.833333, + "confidence": "high", + "pass": true + }, + { + "query": "How much is the vehicle tax for my car with plate 456XYZ?", + "expected": "get_vehicle_tax_info", + "got": "get_vehicle_tax_info", + "cosine_score": 0.6316, + "rrf_score": 1.0, + "confidence": "high", + "pass": true + }, + { + "query": "What is the car tax based on my registration number?", + "expected": "get_vehicle_tax_info", + "got": "get_vehicle_tax_info", + "cosine_score": 0.5134, + "rrf_score": 0.833333, + "confidence": "medium", + "pass": true + }, + { + "query": "Arvuta s\u00f5idukimaks registreerimisnumbri 123ABC alusel", + "expected": "get_vehicle_tax_info", + "got": "get_vehicle_tax_info", + "cosine_score": 0.8383, + "rrf_score": 0.833333, + "confidence": "high", + "pass": true + }, + { + "query": "Kui suur on minu auto maks numbrim\u00e4rgi 456XYZ j\u00e4rgi?", + "expected": "get_vehicle_tax_info", + "got": "get_vehicle_tax_info", + "cosine_score": 0.824, + "rrf_score": 0.642857, + "confidence": "high", + "pass": true + }, + { + "query": "Mis on mootors\u00f5idukimaks minu auto registreerimisnumbri alusel?", + "expected": "get_vehicle_tax_info", + "got": "get_vehicle_tax_info", + "cosine_score": 0.6378, + "rrf_score": 0.666667, + "confidence": "high", + "pass": true + }, + { + "query": "Show me the latest parliament voting records in Estonia", + "expected": "get_parliament_votings", + "got": "get_parliament_votings", + "cosine_score": 0.6093, + "rrf_score": 0.5, + "confidence": "high", + "pass": true + }, + { + "query": "What did the Riigikogu vote on recently?", + "expected": "get_parliament_votings", + "got": "get_parliament_votings", + "cosine_score": 0.7187, + "rrf_score": 0.5, + "confidence": "medium", + "pass": true + }, + { + "query": "Retrieve parliamentary voting decisions from the Estonian parliament", + "expected": "get_parliament_votings", + "got": "get_parliament_votings", + "cosine_score": 0.6544, + "rrf_score": 0.5, + "confidence": "high", + "pass": true + }, + { + "query": "What laws were voted on in the Estonian parliament?", + "expected": "get_parliament_votings", + "got": "get_parliament_votings", + "cosine_score": 0.5905, + "rrf_score": 0.5, + "confidence": "medium", + "pass": true + }, + { + "query": "N\u00e4ita Riigikogu viimaseid h\u00e4\u00e4letusprotokolle", + "expected": "get_parliament_votings", + "got": "get_parliament_votings", + "cosine_score": 0.6515, + "rrf_score": 0.642857, + "confidence": "medium", + "pass": true + }, + { + "query": "Mille \u00fcle h\u00e4\u00e4letas Riigikogu hiljuti?", + "expected": "get_parliament_votings", + "got": "get_parliament_votings", + "cosine_score": 0.7685, + "rrf_score": 0.666667, + "confidence": "high", + "pass": true + }, + { + "query": "Milliseid seadusi h\u00e4\u00e4letati Eesti parlamendis?", + "expected": "get_parliament_votings", + "got": "get_parliament_votings", + "cosine_score": 0.6675, + "rrf_score": 0.5, + "confidence": "high", + "pass": true + }, + { + "query": "How often do Estonian parliament members attend sessions?", + "expected": "get_parliament_participation_stats", + "got": "get_parliament_participation_stats", + "cosine_score": 0.5247, + "rrf_score": 0.5, + "confidence": "medium", + "pass": true + }, + { + "query": "Show me parliament member attendance statistics", + "expected": "get_parliament_participation_stats", + "got": "get_parliament_participation_stats", + "cosine_score": 0.5584, + "rrf_score": 0.5, + "confidence": "medium", + "pass": true + }, + { + "query": "Which MPs have the best attendance record in the Riigikogu?", + "expected": "get_parliament_participation_stats", + "got": "get_parliament_participation_stats", + "cosine_score": 0.7052, + "rrf_score": 0.75, + "confidence": "high", + "pass": true + }, + { + "query": "Kui tihti osalevad Riigikogu liikmed istungitel?", + "expected": "get_parliament_participation_stats", + "got": "get_parliament_participation_stats", + "cosine_score": 0.6471, + "rrf_score": 0.7, + "confidence": "high", + "pass": true + }, + { + "query": "N\u00e4ita Riigikogu liikmete kohaloleku statistikat", + "expected": "get_parliament_participation_stats", + "got": "get_parliament_participation_stats", + "cosine_score": 0.7678, + "rrf_score": 0.833333, + "confidence": "high", + "pass": true + }, + { + "query": "Millistel saadikutel on Riigikogu parim kohalolekurekord?", + "expected": "get_parliament_participation_stats", + "got": "get_parliament_participation_stats", + "cosine_score": 0.631, + "rrf_score": 0.642857, + "confidence": "high", + "pass": true + }, + { + "query": "Show me a list of active citizen initiatives in Estonia", + "expected": "get_initiatives", + "got": "get_initiatives", + "cosine_score": 0.6595, + "rrf_score": 0.5, + "confidence": "medium", + "pass": true + }, + { + "query": "What public initiatives are currently available?", + "expected": "get_initiatives", + "got": null, + "cosine_score": null, + "rrf_score": null, + "confidence": null, + "pass": false + }, + { + "query": "List all citizen initiatives on rahvaalgatus.ee", + "expected": "get_initiatives", + "got": "get_initiatives", + "cosine_score": 0.6632, + "rrf_score": 0.5, + "confidence": "medium", + "pass": true + }, + { + "query": "N\u00e4ita mulle aktiivsete kodanike algatuste nimekirja Eestis", + "expected": "get_initiatives", + "got": "get_initiatives", + "cosine_score": 0.6846, + "rrf_score": 0.666667, + "confidence": "high", + "pass": true + }, + { + "query": "Millised rahvaalgatused on praegu saadaval?", + "expected": "get_initiatives", + "got": "get_initiatives", + "cosine_score": 0.7545, + "rrf_score": 0.5, + "confidence": "high", + "pass": true + }, + { + "query": "Loetle k\u00f5ik algatused rahvaalgatus.ee lehel", + "expected": "get_initiatives", + "got": "get_initiatives", + "cosine_score": 0.5893, + "rrf_score": 0.5, + "confidence": "medium", + "pass": true + }, + { + "query": "Get details about citizen initiative with ID abc123", + "expected": "get_initiative_details", + "got": "get_initiative_details", + "cosine_score": 0.5825, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "Show me more information about a specific public initiative", + "expected": "get_initiative_details", + "got": "get_initiative_details", + "cosine_score": 0.4889, + "rrf_score": 0.5, + "confidence": "medium", + "pass": true + }, + { + "query": "Fetch the details of initiative ID xyz789", + "expected": "get_initiative_details", + "got": "get_initiative_details", + "cosine_score": 0.5759, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "Too andmed kodanike algatuse ID abc123 kohta", + "expected": "get_initiative_details", + "got": "get_initiative_details", + "cosine_score": 0.5205, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "N\u00e4ita mulle \u00fcksikasju konkreetse rahvaalgatuse kohta", + "expected": "get_initiative_details", + "got": "get_initiative_details", + "cosine_score": 0.6357, + "rrf_score": 0.666667, + "confidence": "medium", + "pass": true + }, + { + "query": "Too algatuse ID xyz789 \u00fcksikasjad", + "expected": "get_initiative_details", + "got": "get_initiative_details", + "cosine_score": 0.4259, + "rrf_score": 0.833333, + "confidence": "medium", + "pass": true + }, + { + "query": "What are the latest events related to citizen initiatives?", + "expected": "get_initiative_events", + "got": "get_initiative_events", + "cosine_score": 0.5164, + "rrf_score": 0.5, + "confidence": "medium", + "pass": true + }, + { + "query": "Show me updates and events for public initiatives", + "expected": "get_initiative_events", + "got": "get_initiative_events", + "cosine_score": 0.5302, + "rrf_score": 0.5, + "confidence": "medium", + "pass": true + }, + { + "query": "Are there any new events for citizen initiatives in Estonia?", + "expected": "get_initiative_events", + "got": "get_initiative_events", + "cosine_score": 0.5956, + "rrf_score": 0.5, + "confidence": "medium", + "pass": true + }, + { + "query": "Millised on viimased kodanike algatustega seotud s\u00fcndmused?", + "expected": "get_initiative_events", + "got": "get_initiative_events", + "cosine_score": 0.7619, + "rrf_score": 0.5, + "confidence": "high", + "pass": true + }, + { + "query": "N\u00e4ita rahvaalgatuste uuendusi ja s\u00fcndmusi", + "expected": "get_initiative_events", + "got": "get_initiative_events", + "cosine_score": 0.7217, + "rrf_score": 0.642857, + "confidence": "high", + "pass": true + }, + { + "query": "Kas Eestis on uusi s\u00fcndmusi kodanike algatuste kohta?", + "expected": "get_initiative_events", + "got": "get_initiative_events", + "cosine_score": 0.6662, + "rrf_score": 0.642857, + "confidence": "high", + "pass": true + }, + { + "query": "Search for the address Viru 4 in Tallinn", + "expected": "search_address", + "got": "search_address", + "cosine_score": 0.6338, + "rrf_score": 0.833333, + "confidence": "high", + "pass": true + }, + { + "query": "Find the location of Kadriorg Park in Tallinn", + "expected": "search_address", + "got": "search_address", + "cosine_score": 0.4442, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "Look up an address or place name in Estonia", + "expected": "search_address", + "got": "search_address", + "cosine_score": 0.5577, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "Search for a street address in Tartu", + "expected": "search_address", + "got": "search_address", + "cosine_score": 0.6088, + "rrf_score": 1.0, + "confidence": "high", + "pass": true + }, + { + "query": "Otsi aadressi Viru 4 Tallinnas", + "expected": "search_address", + "got": "search_address", + "cosine_score": 0.7736, + "rrf_score": 0.75, + "confidence": "high", + "pass": true + }, + { + "query": "Leia Kadrioru pargi asukoht Tallinnas", + "expected": "search_address", + "got": "search_address", + "cosine_score": 0.4523, + "rrf_score": 0.5, + "confidence": "medium", + "pass": true + }, + { + "query": "Otsi t\u00e4navaaadress Tartus", + "expected": "search_address", + "got": "search_address", + "cosine_score": 0.6469, + "rrf_score": 0.75, + "confidence": "high", + "pass": true + }, + { + "query": "What is the population of Estonia?", + "expected": "get_population_statistics", + "got": "get_population_statistics", + "cosine_score": 0.4761, + "rrf_score": 0.5, + "confidence": "medium", + "pass": true + }, + { + "query": "Show me population statistics data for Estonia", + "expected": "get_population_statistics", + "got": "get_population_statistics", + "cosine_score": 0.4796, + "rrf_score": 0.5, + "confidence": "medium", + "pass": true + }, + { + "query": "Fetch demographic statistics for Estonia", + "expected": "get_population_statistics", + "got": "get_population_statistics", + "cosine_score": 0.5258, + "rrf_score": 0.5, + "confidence": "medium", + "pass": true + }, + { + "query": "What is the population breakdown by age group in Estonia?", + "expected": "get_population_statistics", + "got": "get_population_statistics", + "cosine_score": 0.5764, + "rrf_score": 0.5, + "confidence": "medium", + "pass": true + }, + { + "query": "Milline on Eesti rahvaarv?", + "expected": "get_population_statistics", + "got": "get_population_statistics", + "cosine_score": 0.5107, + "rrf_score": 0.5, + "confidence": "medium", + "pass": true + }, + { + "query": "N\u00e4ita mulle Eesti rahvastikustatistika andmeid", + "expected": "get_population_statistics", + "got": "get_population_statistics", + "cosine_score": 0.5269, + "rrf_score": 0.833333, + "confidence": "medium", + "pass": true + }, + { + "query": "Milline on Eesti rahvastiku jaotus vanuser\u00fchmade kaupa?", + "expected": "get_population_statistics", + "got": "get_population_statistics", + "cosine_score": 0.5995, + "rrf_score": 0.666667, + "confidence": "medium", + "pass": true + }, + { + "query": "Show me economic statistics for Estonia", + "expected": "get_economic_statistics", + "got": "get_economic_statistics", + "cosine_score": 0.5219, + "rrf_score": 0.5, + "confidence": "medium", + "pass": true + }, + { + "query": "What is the GDP and economic output of Estonia?", + "expected": "get_economic_statistics", + "got": "get_economic_statistics", + "cosine_score": 0.4211, + "rrf_score": 0.5, + "confidence": "medium", + "pass": true + }, + { + "query": "Fetch economic data for Estonia from the statistics office", + "expected": "get_economic_statistics", + "got": "get_economic_statistics", + "cosine_score": 0.5652, + "rrf_score": 0.5, + "confidence": "medium", + "pass": true + }, + { + "query": "N\u00e4ita mulle Eesti majandusstatistikat", + "expected": "get_economic_statistics", + "got": "get_economic_statistics", + "cosine_score": 0.5082, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "Mis on Eesti SKP ja majanduslik toodang?", + "expected": "get_economic_statistics", + "got": "get_economic_statistics", + "cosine_score": 0.5431, + "rrf_score": 0.833333, + "confidence": "medium", + "pass": true + }, + { + "query": "Too majandusandmed Eesti statistikaametist", + "expected": "get_economic_statistics", + "got": "get_economic_statistics", + "cosine_score": 0.4921, + "rrf_score": 0.7, + "confidence": "medium", + "pass": true + }, + { + "query": "What is the unemployment rate in Estonia?", + "expected": "get_labor_statistics", + "got": "get_labor_statistics", + "cosine_score": 0.5905, + "rrf_score": 0.5, + "confidence": "medium", + "pass": true + }, + { + "query": "Show me labor and employment statistics for Estonia", + "expected": "get_labor_statistics", + "got": "get_labor_statistics", + "cosine_score": 0.5207, + "rrf_score": 0.333333, + "confidence": "medium", + "pass": true + }, + { + "query": "How many people are employed in Estonia?", + "expected": "get_labor_statistics", + "got": "get_labor_statistics", + "cosine_score": 0.4705, + "rrf_score": 0.333333, + "confidence": "medium", + "pass": true + }, + { + "query": "Fetch workforce and jobless statistics for Estonia", + "expected": "get_labor_statistics", + "got": "get_labor_statistics", + "cosine_score": 0.5505, + "rrf_score": 0.5, + "confidence": "medium", + "pass": true + }, + { + "query": "Milline on t\u00f6\u00f6tuse m\u00e4\u00e4r Eestis?", + "expected": "get_labor_statistics", + "got": "get_labor_statistics", + "cosine_score": 0.6245, + "rrf_score": 0.5, + "confidence": "high", + "pass": true + }, + { + "query": "N\u00e4ita mulle Eesti t\u00f6\u00f6j\u00f5u ja t\u00f6\u00f6h\u00f5ive statistikat", + "expected": "get_labor_statistics", + "got": "get_labor_statistics", + "cosine_score": 0.556, + "rrf_score": 0.666667, + "confidence": "medium", + "pass": true + }, + { + "query": "Kui palju inimesi t\u00f6\u00f6tab Eestis?", + "expected": "get_labor_statistics", + "got": "get_economic_statistics", + "cosine_score": 0.465, + "rrf_score": 0.5, + "confidence": "medium", + "pass": false + }, + { + "query": "What is the current weather in Tallinn?", + "expected": "get_current_weather", + "got": "get_current_weather", + "cosine_score": 0.7102, + "rrf_score": 0.5, + "confidence": "high", + "pass": true + }, + { + "query": "Show me the current weather conditions in Estonia", + "expected": "get_current_weather", + "got": "get_current_weather", + "cosine_score": 0.5532, + "rrf_score": 0.5, + "confidence": "medium", + "pass": true + }, + { + "query": "What is the temperature right now at the Tallinn weather station?", + "expected": "get_current_weather", + "got": "get_current_weather", + "cosine_score": 0.7024, + "rrf_score": 0.5, + "confidence": "high", + "pass": true + }, + { + "query": "Milline on praegune ilm Tallinnas?", + "expected": "get_current_weather", + "got": "get_current_weather", + "cosine_score": 0.8075, + "rrf_score": 0.5, + "confidence": "medium", + "pass": true + }, + { + "query": "N\u00e4ita mulle praeguseid ilmastikuolusid Eestis", + "expected": "get_current_weather", + "got": "get_current_weather", + "cosine_score": 0.5993, + "rrf_score": 0.5, + "confidence": "medium", + "pass": true + }, + { + "query": "Mis on praegune temperatuur Tallinna ilmajaamas?", + "expected": "get_current_weather", + "got": "get_current_weather", + "cosine_score": 0.8916, + "rrf_score": 0.533333, + "confidence": "high", + "pass": true + }, + { + "query": "What is the weather forecast for Tallinn tomorrow?", + "expected": "get_weather_forecast", + "got": "get_weather_forecast", + "cosine_score": 0.7875, + "rrf_score": 0.5, + "confidence": "high", + "pass": true + }, + { + "query": "Show me the upcoming weather forecast for Tartu", + "expected": "get_weather_forecast", + "got": "get_weather_forecast", + "cosine_score": 0.6849, + "rrf_score": 0.5, + "confidence": "high", + "pass": true + }, + { + "query": "What will the weather be like in Estonia next week?", + "expected": "get_weather_forecast", + "got": "get_weather_forecast", + "cosine_score": 0.6047, + "rrf_score": 0.5, + "confidence": "high", + "pass": true + }, + { + "query": "Give me a weather forecast for the next few days in Estonia", + "expected": "get_weather_forecast", + "got": "get_weather_forecast", + "cosine_score": 0.5652, + "rrf_score": 0.5, + "confidence": "medium", + "pass": true + }, + { + "query": "Milline on ilmaprognoos Tallinnas homme?", + "expected": "get_weather_forecast", + "got": "get_weather_forecast", + "cosine_score": 0.8612, + "rrf_score": 0.666667, + "confidence": "high", + "pass": true + }, + { + "query": "N\u00e4ita mulle Tartu eelseisvat ilmaprognoosi", + "expected": "get_weather_forecast", + "got": "get_weather_forecast", + "cosine_score": 0.77, + "rrf_score": 0.5, + "confidence": "medium", + "pass": true + }, + { + "query": "Milline on ilm Eestis j\u00e4rgmisel n\u00e4dalal?", + "expected": "get_weather_forecast", + "got": "get_weather_forecast", + "cosine_score": 0.6499, + "rrf_score": 0.5, + "confidence": "high", + "pass": true + }, + { + "query": "Who is the Prime Minister of Estonia?", + "expected": null, + "got": null, + "cosine_score": null, + "rrf_score": null, + "confidence": null, + "pass": true + }, + { + "query": "What is the best restaurant in Tallinn?", + "expected": null, + "got": "search_address", + "cosine_score": 0.4746, + "rrf_score": 1.0, + "confidence": "medium", + "pass": true + }, + { + "query": "Tell me a random fact about Estonia", + "expected": null, + "got": null, + "cosine_score": null, + "rrf_score": null, + "confidence": null, + "pass": true + }, + { + "query": "What is the meaning of life?", + "expected": null, + "got": null, + "cosine_score": null, + "rrf_score": null, + "confidence": null, + "pass": true + }, + { + "query": "Book me a flight to London", + "expected": null, + "got": null, + "cosine_score": null, + "rrf_score": null, + "confidence": null, + "pass": true + }, + { + "query": "Can you translate this text to Estonian?", + "expected": null, + "got": null, + "cosine_score": null, + "rrf_score": null, + "confidence": null, + "pass": true + }, + { + "query": "What are the visa requirements to visit Estonia?", + "expected": null, + "got": null, + "cosine_score": null, + "rrf_score": null, + "confidence": null, + "pass": true + }, + { + "query": "How do I apply for an Estonian e-Residency?", + "expected": null, + "got": null, + "cosine_score": null, + "rrf_score": null, + "confidence": null, + "pass": true + }, + { + "query": "What is the history of Tallinn Old Town?", + "expected": null, + "got": null, + "cosine_score": null, + "rrf_score": null, + "confidence": null, + "pass": true + }, + { + "query": "Give me a poem about Estonia", + "expected": null, + "got": null, + "cosine_score": null, + "rrf_score": null, + "confidence": null, + "pass": true + }, + { + "query": "Kes on Eesti peaminister?", + "expected": null, + "got": null, + "cosine_score": null, + "rrf_score": null, + "confidence": null, + "pass": true + }, + { + "query": "Mis on parim restoran Tallinnas?", + "expected": null, + "got": null, + "cosine_score": null, + "rrf_score": null, + "confidence": null, + "pass": true + }, + { + "query": "Mis on elu m\u00f5te?", + "expected": null, + "got": null, + "cosine_score": null, + "rrf_score": null, + "confidence": null, + "pass": true + } +] \ No newline at end of file From 80bfce7ecceedcafbc6a334e60c8d1fc9b991d62 Mon Sep 17 00:00:00 2001 From: nuwangeek Date: Wed, 22 Apr 2026 15:51:01 +0530 Subject: [PATCH 6/6] competed integration of agentic loop with semantic searcher and streaming --- docs/API_TOOL_CALLING.md | 292 ++++++- src/llm_orchestration_service.py | 6 + src/llm_orchestration_service_api.py | 15 + src/models/session_models.py | 8 + src/tool_classifier/agentic_loop.py | 35 +- src/tool_classifier/api_semantic_searcher.py | 35 +- src/tool_classifier/classifier.py | 54 ++ src/tool_classifier/constants.py | 16 +- src/tool_classifier/param_extractor.py | 41 +- .../workflows/api_tool_workflow.py | 321 +++++-- src/utils/language_detector.py | 1 - tests/api_tool_eval/integration-results.json | 191 +++++ .../integration_test_agentic_loop.py | 805 ++++++++++++++++++ 13 files changed, 1677 insertions(+), 143 deletions(-) create mode 100644 tests/api_tool_eval/integration-results.json create mode 100644 tests/api_tool_eval/integration_test_agentic_loop.py diff --git a/docs/API_TOOL_CALLING.md b/docs/API_TOOL_CALLING.md index 7985eb4a..bc0a1160 100644 --- a/docs/API_TOOL_CALLING.md +++ b/docs/API_TOOL_CALLING.md @@ -5,15 +5,17 @@ ## Overview API Tool Calling enables the LLM module to discover and invoke external API endpoints -in response to user queries. endpoints are -registered, semantically indexed in Qdrant, and retrieved at query time using hybrid search. +in response to user queries. Endpoints are registered, semantically indexed in Qdrant, +and retrieved at query time using hybrid search. Once matched, a multi-turn agentic +loop collects all required parameters from the user before the API call is made. -| component | What it does | Status | +| Component | What it does | Status | |---|---|---| | **Indexing pipeline** | Takes an endpoint definition → enriches it with LLM context → stores hybrid vectors in Qdrant | ✅ Complete | | **Tool classifier** | At query time, routes to the best matching endpoint via hybrid search + LLM disambiguation | ✅ Complete | -| **Workflow executor** | Surfaces the matched endpoint; full agentic loop (param collection → API call) planned | 🔧 Partial (Task 10) | +| **Agentic loop** | Multi-turn parameter collection with session persistence, language-aware clarifying questions, param correction, continuation prompt, and intent-switch detection | ✅ Complete | +| **API caller** | Execute the collected params against the real API endpoint and format the response | 🔧 Planned (next task) | --- @@ -35,6 +37,10 @@ APISemanticSearcher (src/tool_classifier/api_semantic_searcher.py) ToolClassifier._try_api_tool_classification() ↓ ClassificationResult(workflow=API_TOOL_CALLING) APIToolWorkflowExecutor (src/tool_classifier/workflows/api_tool_workflow.py) + ↓ multi-turn param collection +AgenticLoop (src/tool_classifier/agentic_loop.py) + ↓ session state +APIToolSessionStore (Redis, keyed by chat_id, 30-min TTL) ``` --- @@ -380,51 +386,281 @@ Defined in [src/tool_classifier/workflows/api_tool_workflow.py](../src/tool_clas Handles `WorkflowType.API_TOOL_CALLING` after `ToolClassifier.classify()` has set `matched_endpoint` in the context dict. -**Current behaviour (Task 4.1):** +**Responsibilities:** -Reads `context["matched_endpoint"]` and returns a simple confirmation response: +- **Turn 1 (new session):** reads `context["matched_endpoint"]`, creates a new + `APIToolSession` in Redis, runs the first agentic loop turn. +- **Turn 2-N (resume):** loads the existing session from Redis, runs the next turn. +- **Fast path:** if the endpoint has no required params, immediately returns the + completed JSON without starting a session. +- **Completion:** when all params are collected, deletes the session and returns a + JSON response with `status=params_collected`. +- **Max turns:** deletes the session and returns `None` to trigger RAG fallback. +- **Streaming:** wraps the short clarifying-question response in a single SSE frame + + `END` marker. +**Completed response format:** + +```json +{ + "status": "params_collected", + "endpoint": { "name": "get_public_holidays" }, + "collected_params": { + "countryIsoCode": "EE", + "validFrom": "2026-01-01", + "validTo": "2026-12-31" + } +} +``` + +The actual API call and response formatting are handled by the next planned task. + +--- + +## Part 3 — Agentic Loop (Multi-Turn Parameter Collection) + +### Overview + +Defined in [src/tool_classifier/agentic_loop.py](../src/tool_classifier/agentic_loop.py). + +`AgenticLoop` is **stateless** — it carries no internal state between HTTP requests. +All state is passed in as arguments (loaded from Redis by the workflow executor before +calling `run_turn`) and saved back to Redis inside `run_turn` before returning. + +### Session Model: `APIToolSession` + +Defined in [src/models/session_models.py](../src/models/session_models.py). + +Stored in Redis keyed by `chat_id` with a **30-minute sliding TTL**. + +| Field | Type | Description | +|---|---|---| +| `chat_id` | str | Unique conversation identifier | +| `state` | str | Current state (`collecting_params`, etc.) | +| `selected_endpoint` | dict | Full endpoint payload from Qdrant | +| `collected_params` | dict | Parameters collected so far | +| `turn_count` | int | Number of turns elapsed | +| `max_turns` | int | Max turns before fallback (default: 5) | +| `awaiting_continuation` | bool | True when continuation prompt has been shown | +| `detected_language` | str | Language from first message (`en`, `et`, `ru`) — persisted so all clarifying questions use the same language | + +### Turn Flow + +``` +APIToolWorkflowExecutor._run() + │ + ├─ Load session from Redis (or create new) + │ + └─ AgenticLoop.run_turn( + user_message, conversation_history, + params_schema, collected_params, + turn_count, max_turns, awaiting_continuation, + session_language + ) + │ + ├─ AWAITING_CONTINUATION_DECISION? + │ yes → parse yes/no from user_message + │ yes → clear flag, continue collecting + │ no → return MAX_TURNS_REACHED (RAG fallback) + │ + ├─ ParamExtractionModule.forward() + │ → DSPy extracts params from user_message + conversation_history + │ → uses session_language for all questions + │ → new values OVERWRITE old (allows corrections) + │ + ├─ All required params present? → COMPLETED + │ + ├─ turn_count reached CONTINUATION_TURN (default: 3)? + │ → set awaiting_continuation=True + │ → return AWAITING_CONTINUATION_DECISION + │ → question = localized CONTINUATION_QUESTION (EN/ET/RU) + │ + └─ else → generate clarifying question for next missing param + → return NEEDS_INPUT + │ + └─ Save updated session to Redis +``` + +### Key Behaviours + +**Language persistence:** +The language is detected once from the user's first message and stored in +`APIToolSession.detected_language`. All subsequent clarifying questions and the +continuation prompt are generated in that language, even when follow-up replies +like "yes" or "2026-01-01" are too short to re-detect reliably. + +Supported: `en` (default), `et` (Estonian), `ru` (Russian). + +**Parameter correction:** +If the user says "No, use Russia instead of Estonia", the extractor overwrites the +previously collected `countryIsoCode` value. There is no guard preventing +re-extraction of already-collected params — new values always win. + +**Continuation prompt:** +After `CONTINUATION_TURN` turns without completing, the loop asks the user whether +to continue. If the user says no (or anything not in the yes-list), the session is +abandoned and the request falls back to the RAG workflow. + +Localized continuation questions are defined in +[src/tool_classifier/constants.py](../src/tool_classifier/constants.py): +`CONTINUATION_QUESTION`, `CONTINUATION_QUESTION_ET`, `CONTINUATION_QUESTION_RU`. + +**History isolation:** +On turn 0 (first turn of a new session), `conversation_history=[]` is passed to the +extractor regardless of what the API sends. This prevents parameter values from a +previous completed session from being re-used for the new request. + +**Constants** (in `src/tool_classifier/constants.py`): + +| Constant | Value | Description | +|---|---|---| +| `CONTINUATION_TURN` | `3` | Turn at which the continuation prompt is shown | + +--- + +## Part 4 — Session Management & Intent Switch Detection + +### `APIToolSessionStore` + +Defined in [src/utils/api_tool_session_store.py](../src/utils/api_tool_session_store.py). + +Redis-backed store. Key format: `session:{chat_id}`. TTL resets on every `update()`. + +Operations: `save()`, `get()`, `update()`, `delete()`. + +### Session Lifecycle + +``` +Turn 1: new query matches API tool endpoint + → session CREATED (state=collecting_params) + → clarifying question returned + +Turn 2-N: user replies + → session LOADED → loop runs → session UPDATED + +Final turn: all params collected + → session DELETED + → completed JSON returned + +OR: max turns reached / user says "no" to continuation + → session DELETED + → None returned → RAG fallback ``` -**{name}**: {description} -URL: {url} +### Intent Switch Detection + +Defined in `ToolClassifier.classify()` — the session-resume short-circuit block. + +Before resuming an active session, the classifier runs `_try_api_tool_classification()` +on the new message. If it matches a **different** endpoint with sufficient confidence, +the old session is abandoned and the new query starts fresh: + +```python +new_api_match = await self._try_api_tool_classification(query, request) +if ( + new_api_match is not None + and new_api_match.metadata["matched_endpoint"]["name"] != endpoint_name +): + await session_store.delete(request.chatId) + return new_api_match # start new session for different endpoint ``` -**Planned (Task 10):** Full agentic loop — -session management → parameter collection dialog → external API call → response formatting. +### Test Endpoint Behaviour (`/orchestrate/test`) + +The test endpoint hardcodes `chatId="test-session"` for all requests. Because every +test user shares this ID, any incomplete session would be resumed by the next +unrelated test query. + +**Fix:** the test endpoint deletes `"test-session"` from Redis at the **start** of +every request, before classification runs. This makes each test query a fresh +single-turn request. + +**Trade-off:** multi-turn API tool flows cannot be tested via the test-LLM page. +The session is wiped before turn 2 can use it. To test multi-turn flows, use the +production `/orchestrate/stream` endpoint (which uses unique `chatId` per tab) or +the integration test script. --- ### End-to-End Flow (Query Time) ``` -User: "What are the public holidays in Estonia?" +Turn 1 — User: "What are the public holidays in Estonia?" │ ▼ ToolClassifier.classify() │ + ├─ No active session in Redis for this chat_id ├─ Dense search (intent_collections) → low cosine → below threshold - │ └─ _try_api_tool_classification() - │ └─ APISemanticSearcher.search() - │ - ├─ Dense: get_national_holidays cosine=0.87 - ├─ Hybrid: get_national_holidays ranked #1 (RRF) - ├─ effective_gap large → confidence="high" - └─ return [APIToolSearchResult(name="get_national_holidays", ...)] - │ - └─ ClassificationResult( - workflow=API_TOOL_CALLING, - metadata={"matched_endpoint": {...}} - ) + ├─ Dense: get_public_holidays cosine=0.87 → high confidence + └─ return [APIToolSearchResult(name="get_public_holidays", ...)] + └─ ClassificationResult(workflow=API_TOOL_CALLING, metadata={matched_endpoint: {...}}) │ ▼ -ToolClassifier._execute_with_fallback_async() +APIToolWorkflowExecutor._run() + ├─ No existing session → create new APIToolSession (turn_count=0, language=en) + └─ AgenticLoop.run_turn(turn_count=0, history=[]) + ├─ ParamExtractionModule: no params in "What are the public holidays in Estonia?" + │ but countryIsoCode=EE can be inferred → extracted + ├─ Missing: validFrom, validTo + └─ NEEDS_INPUT → "Which date range would you like? (validFrom, validTo)" │ - └─ APIToolWorkflowExecutor.execute_async(context={"matched_endpoint": {...}}) - │ - └─ OrchestrationResponse(content="**get_national_holidays**: ...") + Session saved to Redis + ▼ +Bot: "Which date range would you like? Please provide validFrom and validTo (YYYY-MM-DD)." + +--- + +Turn 2 — User: "This year, 2026-01-01 to 2026-12-31" + │ + ▼ +ToolClassifier.classify() + ├─ Active session found for chat_id → run intent-switch check + ├─ _try_api_tool_classification("This year, 2026-01-01 to 2026-12-31") + │ → cosine=0.12 < threshold → no new API tool match + └─ Same endpoint → resume session → ClassificationResult(reason=active_session_resume) + │ + ▼ +APIToolWorkflowExecutor._run() + └─ AgenticLoop.run_turn(turn_count=1, collected_params={countryIsoCode: "EE"}) + ├─ ParamExtractionModule: extracts validFrom=2026-01-01, validTo=2026-12-31 + ├─ All required params present + └─ COMPLETED + │ + Session DELETED from Redis + ▼ +Bot: {"status": "params_collected", "endpoint": {"name": "get_public_holidays"}, "collected_params": {"countryIsoCode": "EE", "validFrom": "2026-01-01", "validTo": "2026-12-31"}} ``` ---- \ No newline at end of file +--- + +## Part 5 — Integration Testing + +### Test Script + +Defined in [tests/api_tool_eval/integration_test_agentic_loop.py](../tests/api_tool_eval/integration_test_agentic_loop.py). + +Runs end-to-end against the live service at `http://localhost:8100` via `/orchestrate`. +Each scenario uses a unique `chatId` (UUID) so sessions are fully isolated. + +```bash +uv run --no-project --with requests python tests/api_tool_eval/integration_test_agentic_loop.py \ + --no-fail-fast \ + --output tests/api_tool_eval/integration-results.json +``` + +### Covered Scenarios + +| # | Scenario | Turns | What it validates | +|---|---|---|---| +| 1 | Single-turn complete | 1 | Vehicle tax with plate number in first message → immediate completion | +| 2 | Multi-turn EN | 2 | Public holidays, country extracted turn 1, dates provided turn 2 | +| 3 | Multi-turn ET | 2 | School holidays in Estonian → language-aware classification | +| 4 | No-params fast path | 1 | Parliament votings endpoint has no required params → instant completion | +| 5 | Address search | 2 | Two-turn address lookup | +| 6 | Electricity prices | 2 | Datetime params across two turns | +| 7 | Session isolation | 2 | Two different chat IDs — no param leak between sessions | +| 8 | AWAITING_CONTINUATION → yes | 4+ | User says "yes" at continuation prompt → loop resumes | +| 9 | MAX_TURNS_REACHED | 5+ | User never provides params → falls back to RAG | \ No newline at end of file diff --git a/src/llm_orchestration_service.py b/src/llm_orchestration_service.py index 91629baa..070ff241 100644 --- a/src/llm_orchestration_service.py +++ b/src/llm_orchestration_service.py @@ -136,6 +136,12 @@ def __init__(self) -> None: # This allows components to be initialized per-request with proper context self.tool_classifier = None + # Redis-backed session store for API Tool Calling agentic loop. + # Set to None here; the FastAPI lifespan injects the live store after + # Redis initialises (app.state.orchestration_service.session_store = ...). + # Workflow executors access it via self.orchestration_service.session_store. + self.session_store: Any = None + # Shared BM25 search index pre-warmed at startup. # Populated by _prewarm_shared_bm25() which is called from the FastAPI # lifespan so it runs inside the async event loop. Until then it is None diff --git a/src/llm_orchestration_service_api.py b/src/llm_orchestration_service_api.py index ddd66a9a..6ecd0388 100644 --- a/src/llm_orchestration_service_api.py +++ b/src/llm_orchestration_service_api.py @@ -96,6 +96,14 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: logger.warning(f"Redis session store unavailable, continuing without it: {e}") app.state.session_store = None + # Expose session_store on the orchestration service so workflow executors + # (e.g. APIToolWorkflowExecutor) can reach it via self.orchestration_service. + if ( + hasattr(app.state, "orchestration_service") + and app.state.orchestration_service is not None + ): + app.state.orchestration_service.session_store = app.state.session_store + yield # Shutdown @@ -383,6 +391,13 @@ async def test_orchestrate_llm_request( else None, ) + # test-LLM is single-turn only (no conversationHistory, no multi-turn loops). + # Clear any stale API tool session so each request starts fresh and never + # accidentally resumes a parameter-collection loop from a previous test query. + session_store = getattr(http_request.app.state, "session_store", None) + if session_store is not None: + await session_store.delete("test-session") + logger.info(f"This is full request constructed for testing: {full_request}") # Process the request using the same logic diff --git a/src/models/session_models.py b/src/models/session_models.py index 8f36fd23..0620b62a 100644 --- a/src/models/session_models.py +++ b/src/models/session_models.py @@ -41,3 +41,11 @@ class APIToolSession(BaseModel): "to the RAG workflow." ), ) + detected_language: str = Field( + default="en", + description=( + "Language detected from the user's first message ('en', 'et', 'ru'). " + "Persisted so all subsequent clarifying questions use the same language, " + "even when follow-up messages are too short to reliably re-detect." + ), + ) diff --git a/src/tool_classifier/agentic_loop.py b/src/tool_classifier/agentic_loop.py index fd5ae1d6..f2b6d6bf 100644 --- a/src/tool_classifier/agentic_loop.py +++ b/src/tool_classifier/agentic_loop.py @@ -6,12 +6,23 @@ from loguru import logger from src.utils.api_tool_session_store import APIToolSessionStore -from tool_classifier.constants import CONTINUATION_QUESTION, CONTINUATION_TURN +from tool_classifier.constants import ( + CONTINUATION_QUESTION, + CONTINUATION_QUESTION_ET, + CONTINUATION_QUESTION_RU, + CONTINUATION_TURN, +) from tool_classifier.enums import AgenticLoopStatus from tool_classifier.models import AgenticLoopResult from tool_classifier.param_extractor import ParamExtractionModule -# Normalised user responses that indicate the user wants to keep collecting params. +_CONTINUATION_QUESTIONS: dict[str, str] = { + "en": CONTINUATION_QUESTION, + "et": CONTINUATION_QUESTION_ET, + "ru": CONTINUATION_QUESTION_RU, +} + + _YES_RESPONSES = frozenset( { "yes", @@ -94,6 +105,7 @@ async def run_turn( max_turns: int = 5, awaiting_continuation: bool = False, continuation_turn: int = CONTINUATION_TURN, + session_language: str = "en", ) -> AgenticLoopResult: """Process one user turn of the parameter-collection loop. @@ -193,6 +205,7 @@ async def run_turn( params_schema, conversation_history, collected_params, + session_language, ) except Exception as exc: logger.error( @@ -218,10 +231,13 @@ async def run_turn( turn_count=updated_turn_count, ) - # Step 3 — Merge: prior values take precedence (already_collected is authoritative) + # Step 3 — Merge: newly extracted values override prior ones so the user + # can correct a value they provided in an earlier turn (e.g. "actually, + # make that Russia instead of Estonia"). Prior values are kept only for + # params the extractor did NOT mention in this turn. merged_params: Dict[str, Any] = { - **extraction["extracted_params"], **collected_params, + **extraction["extracted_params"], } # Step 4 — Completeness check @@ -263,13 +279,16 @@ async def run_turn( turn_count, chat_id, ) + continuation_q = _CONTINUATION_QUESTIONS.get( + session_language, CONTINUATION_QUESTION + ) await self._save_session( chat_id, merged_params, updated_turn_count, awaiting_continuation=True ) return AgenticLoopResult( status=AgenticLoopStatus.AWAITING_CONTINUATION_DECISION, collected_params=merged_params, - clarifying_question=CONTINUATION_QUESTION, + clarifying_question=continuation_q, turn_count=updated_turn_count, ) @@ -298,6 +317,12 @@ async def _save_session( A missing or unavailable session is logged but never raises. """ try: + if self._session_store is None: + logger.debug( + "AgenticLoop: session store unavailable — skipping save for chat_id={}", + chat_id, + ) + return await self._session_store.update( chat_id, collected_params=collected_params, diff --git a/src/tool_classifier/api_semantic_searcher.py b/src/tool_classifier/api_semantic_searcher.py index 9b197d40..7d5c5f00 100644 --- a/src/tool_classifier/api_semantic_searcher.py +++ b/src/tool_classifier/api_semantic_searcher.py @@ -544,23 +544,30 @@ async def _hybrid_search( Returns deduplicated results by endpoint_id, sorted by RRF score. """ try: - # Verify collection is non-empty before searching - collection_info = await self._qdrant_client.get( - f"/collections/{API_TOOL_COLLECTION}" - ) - if collection_info.status_code == 200: - points_count = ( - collection_info.json().get("result", {}).get("points_count", 0) + # Verify collection is non-empty before searching. + # This check is only an optimization: if it fails, continue with the + # actual search request instead of failing closed. + try: + collection_info = await self._qdrant_client.get( + f"/collections/{API_TOOL_COLLECTION}" ) - if points_count == 0: - logger.info("APISemanticSearcher: api_tool_collection is empty") - return [] - else: + if collection_info.status_code == 200: + points_count = ( + collection_info.json().get("result", {}).get("points_count", 0) + ) + if points_count == 0: + logger.info("APISemanticSearcher: api_tool_collection is empty") + return [] + else: + logger.warning( + f"APISemanticSearcher: Could not verify collection: " + f"HTTP {collection_info.status_code}; continuing with search" + ) + except Exception as e: logger.warning( - f"APISemanticSearcher: Could not verify collection: " - f"HTTP {collection_info.status_code}" + f"APISemanticSearcher: Collection verification failed: {e}; " + f"continuing with search" ) - return [] # Build prefetch + RRF payload search_payload: Dict[str, Any] = { diff --git a/src/tool_classifier/classifier.py b/src/tool_classifier/classifier.py index 06be08c7..6326486a 100644 --- a/src/tool_classifier/classifier.py +++ b/src/tool_classifier/classifier.py @@ -152,6 +152,60 @@ async def classify( logger.info(f"Classifying query: {query[:100]}...") try: + # Pre-classification: if an API tool session already exists for this + # chat_id, the user is responding to a param-collection question. + # Short-circuit directly to API_TOOL_CALLING — no need to re-classify. + if FeatureFlags.API_TOOL_CALLING_WORKFLOW_ENABLED and request is not None: + session_store = getattr( + self.orchestration_service, "session_store", None + ) + if session_store is not None: + existing_session = await session_store.get(request.chatId) + if existing_session is not None: + endpoint_name = ( + existing_session.selected_endpoint.get("name") + if existing_session.selected_endpoint + else "unknown" + ) + + # Before resuming, check if the user's new message is a + # strong match for a DIFFERENT endpoint (intent switch). + # If so, abandon the old session and start fresh rather + # than treating the new query as a param-collection reply. + new_api_match = await self._try_api_tool_classification( + query, request + ) + if ( + new_api_match is not None + and new_api_match.metadata.get("matched_endpoint", {}).get( + "name" + ) + != endpoint_name + ): + logger.info( + f"[{request.chatId}] Intent switch detected: " + f"active session={endpoint_name!r}, " + f"new match={new_api_match.metadata.get('matched_endpoint', {}).get('name')!r} " + f"— abandoning old session" + ) + await session_store.delete(request.chatId) + return new_api_match + + logger.info( + f"[{request.chatId}] Active API tool session found " + f"(endpoint={endpoint_name!r}) " + f"— short-circuiting to API_TOOL_CALLING" + ) + return ClassificationResult( + workflow=WorkflowType.API_TOOL_CALLING, + confidence=1.0, + metadata={ + "reason": "active_session_resume", + "matched_endpoint": existing_session.selected_endpoint, + }, + reasoning="Resuming active API tool parameter-collection session", + ) + if not FeatureFlags.SERVICE_WORKFLOW_ENABLED: logger.info( "SERVICE_WORKFLOW_ENABLED=false - skipping standard service search" diff --git a/src/tool_classifier/constants.py b/src/tool_classifier/constants.py index 98181afe..9155a2e7 100644 --- a/src/tool_classifier/constants.py +++ b/src/tool_classifier/constants.py @@ -151,7 +151,19 @@ CONTINUATION_QUESTION = ( "I still need a bit more information, but we've been at this for a while. " - "Would you like to keep going and answer a few more questions, " - "or would you prefer to stop and get a general answer instead? (yes / no)" + "Would you like to keep going and answer a few more questions " + "(yes / no)" ) """Yes/no question shown to the user when the continuation threshold is reached.""" + +CONTINUATION_QUESTION_ET = ( + "Mul on vaja veel natuke lisateavet, kuid oleme selle kallal juba mõnda aega töötanud. " + "Kas soovite jätkata ja vastata veel mõnele küsimusele? (jah / ei)" +) +"""Estonian version of the continuation question.""" + +CONTINUATION_QUESTION_RU = ( + "Мне нужно ещё немного информации, но мы уже некоторое время занимаемся этим. " + "Хотите ли вы продолжить и ответить ещё на несколько вопросов? (да / нет)" +) +"""Russian version of the continuation question.""" diff --git a/src/tool_classifier/param_extractor.py b/src/tool_classifier/param_extractor.py index aaaa40aa..c6d1e1ef 100644 --- a/src/tool_classifier/param_extractor.py +++ b/src/tool_classifier/param_extractor.py @@ -26,12 +26,17 @@ class ParamExtractionSignature(dspy.Signature): CRITICAL LANGUAGE RULE: - Understand Estonian, English, and Russian input - - Generate clarifying_question in the SAME language as the user_message + - Generate clarifying_question in the language specified by session_language + - IGNORE the language of the current user_message for output language decisions — + short follow-up messages ("I'm not sure", "2026-01-01") are unreliable indicators. + Always use session_language. Extraction rules: - - Extract values for parameters listed in params_schema that are not yet in already_collected - - Search BOTH user_message AND conversation_history for values - - Do NOT re-extract parameters already present in already_collected + - Extract values for ALL parameters listed in params_schema that appear in user_message + or conversation_history, regardless of whether they are already in already_collected + - If the user explicitly provides a new or corrected value for a parameter that is + already in already_collected, still extract the new value — it will override the old one + - Only skip extraction for a param if the user has NOT mentioned it at all in this turn - Validate types: dates must be ISO 8601 (YYYY-MM-DD), integers must be whole numbers, numbers must be numeric, booleans must be true or false @@ -59,11 +64,23 @@ class ParamExtractionSignature(dspy.Signature): conversation_history: str = dspy.InputField( desc="Recent conversation turns formatted as 'role: message', one per line" ) + session_language: str = dspy.InputField( + desc=( + "ISO language code for the response language detected from the user's " + "first message: 'en' (English), 'et' (Estonian), 'ru' (Russian). " + "Always generate clarifying_question in this language." + ) + ) params_schema: str = dspy.InputField( desc='JSON array of parameter schemas: [{"name": str, "type": str, "required": bool, "description": str}]' ) already_collected: str = dspy.InputField( - desc="JSON object of already-collected parameter values: {param_name: value}" + desc=( + "JSON object of parameter values collected in prior turns: {param_name: value}. " + "Use this as context to understand what has already been provided. " + "If the user explicitly mentions a new value for a param already here, " + "still extract the new value — corrections are allowed." + ) ) extracted_params: str = dspy.OutputField( @@ -91,6 +108,7 @@ def forward( params_schema: List[Dict[str, Any]], conversation_history: Optional[List[Dict[str, Any]]] = None, already_collected: Optional[Dict[str, Any]] = None, + session_language: str = "en", ) -> ParamExtractionResult: """ Extract parameter values from user message and conversation history. @@ -100,6 +118,8 @@ def forward( params_schema: List of parameter schema dicts with name, type, required, description conversation_history: Recent conversation messages (optional) already_collected: Parameter values collected in prior turns (optional) + session_language: Language code detected on turn 0 ('en', 'et', 'ru'). + All clarifying questions will be generated in this language. Returns: ParamExtractionResult with extracted_params, missing_required, clarifying_question @@ -115,6 +135,7 @@ def forward( result = self.extractor( user_message=user_message, conversation_history=history_text, + session_language=session_language, params_schema=params_schema_json, already_collected=already_collected_json, ) @@ -139,9 +160,6 @@ def forward( type_invalid_params: List[str] = [] for param_name, raw_value in extracted_raw.items(): - if param_name in already_collected: - # Prior turns are authoritative — discard any LLM re-output - continue schema_entry = schema_map.get(param_name) if schema_entry is None: # Param not in schema — skip silently @@ -157,9 +175,10 @@ def forward( ) type_invalid_params.append(param_name) - # Re-derive missing required params after type validation - # already_collected is authoritative: its values must not be overwritten - all_collected = {**validated_params, **already_collected} + # Re-derive missing required params after type validation. + # validated_params (current turn) takes precedence over already_collected + # so that explicit user corrections override prior values. + all_collected = {**already_collected, **validated_params} missing_required: List[str] = [ p["name"] for p in params_schema diff --git a/src/tool_classifier/workflows/api_tool_workflow.py b/src/tool_classifier/workflows/api_tool_workflow.py index 9403a10a..82849271 100644 --- a/src/tool_classifier/workflows/api_tool_workflow.py +++ b/src/tool_classifier/workflows/api_tool_workflow.py @@ -1,79 +1,78 @@ -"""API Tool Calling Workflow Executor — Layer 2 of the classification chain. +"""API Tool Calling Workflow Executor — Layer 2 of the classification chain.""" -This is the Task 4.1 minimal implementation that surfaces the matched API endpoint -to the user. The full agentic loop (parameter collection → API call → response -formatting) will be implemented in Task 10. -""" - -from typing import Any, AsyncIterator, Dict, Optional +import json +from typing import Any, AsyncIterator, Dict, List, Optional from loguru import logger from models.request_models import OrchestrationRequest, OrchestrationResponse +from models.session_models import APIToolSession +from tool_classifier.agentic_loop import AgenticLoop from tool_classifier.base_workflow import BaseWorkflow +from tool_classifier.enums import AgenticLoopStatus +from tool_classifier.param_extractor import ParamExtractionModule class APIToolWorkflowExecutor(BaseWorkflow): """Executes API Tool Calling workflow (Layer 2). Handles queries that matched an API endpoint in api_tool_collection. - Reads the matched endpoint from context (populated by ToolClassifier.classify()) - and returns it as a response. - Task 10 will replace the placeholder response body with the full agentic loop: - session management → parameter collection → external API call → response formatting. + On the first turn for a chat_id the matched endpoint is read from context + (populated by ToolClassifier.classify()). Subsequent turns resume from the + persisted Redis session — context["matched_endpoint"] is ignored once a + session exists. + + The executor manages the agentic loop lifecycle: + - creates the session on turn 1 + - resumes it on turns 2-N + - deletes it on COMPLETED or MAX_TURNS_REACHED + + When all required params are collected (COMPLETED) the response content is a + JSON string with keys ``status``, ``endpoint``, and ``collected_params``. + The API call and response formatting are handled by the next task. """ def __init__(self, orchestration_service: Optional[Any] = None) -> None: - """Initialize API tool calling workflow. - - Args: - orchestration_service: Reference to LLMOrchestrationService — required - for streaming mode (format_sse). Optional for non-streaming. - """ self.orchestration_service = orchestration_service - async def execute_async( - self, - request: OrchestrationRequest, - context: Dict[str, Any], - time_metric: Optional[Dict[str, float]] = None, - ) -> Optional[OrchestrationResponse]: - """Execute API tool calling workflow in non-streaming mode. - - Args: - request: Orchestration request. - context: Must contain "matched_endpoint" dict from APISemanticSearcher. - time_metric: Optional timing dict for step tracking. - - Returns: - OrchestrationResponse with matched endpoint info, or None if no - endpoint in context (triggers fallback to next layer). - """ - chat_id = request.chatId - endpoint = context.get("matched_endpoint") + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ - if not endpoint: - logger.warning( - f"[{chat_id}] APIToolWorkflow: no matched_endpoint in context — falling back" - ) + def _get_session_store(self) -> Optional[Any]: + """Return the session store from the orchestration service, or None.""" + if self.orchestration_service is None: return None + return getattr(self.orchestration_service, "session_store", None) - name = endpoint.get("name", "unknown") - description = endpoint.get("description", "") - url = endpoint.get("url", "N/A") - confidence = endpoint.get("confidence", "medium") - cosine_score = endpoint.get("cosine_score", 0.0) - - logger.info( - f"[{chat_id}] APIToolWorkflow: matched endpoint={name!r} " - f"(confidence={confidence}, cosine={cosine_score:.4f})" + def _build_agentic_loop(self, session_store: Any) -> AgenticLoop: + """Construct a fresh AgenticLoop for one request.""" + return AgenticLoop( + session_store=session_store, + param_extractor=ParamExtractionModule(), ) - # TODO (Task 10): Replace with full agentic loop — - # load/create session → run param extraction → call external API → format response - content = f"**{name}**: {description}\n\nURL: {url}" + @staticmethod + def _required_params(params: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + return [p for p in params if isinstance(p, dict) and p.get("required", False)] + @staticmethod + def _build_completed_response( + chat_id: str, + endpoint: Dict[str, Any], + collected_params: Dict[str, Any], + ) -> OrchestrationResponse: + content = json.dumps( + { + "status": "params_collected", + "endpoint": { + "name": endpoint.get("name"), + }, + "collected_params": collected_params, + }, + ensure_ascii=False, + ) return OrchestrationResponse( chatId=chat_id, llmServiceActive=True, @@ -82,55 +81,213 @@ async def execute_async( content=content, ) - async def execute_streaming( + @staticmethod + def _build_question_response(chat_id: str, question: str) -> OrchestrationResponse: + return OrchestrationResponse( + chatId=chat_id, + llmServiceActive=True, + questionOutOfLLMScope=False, + inputGuardFailed=False, + content=question, + ) + + # ------------------------------------------------------------------ + # Core loop handler — shared by async and streaming paths + # ------------------------------------------------------------------ + + async def _run( self, request: OrchestrationRequest, context: Dict[str, Any], - time_metric: Optional[Dict[str, float]] = None, - ) -> Optional[AsyncIterator[str]]: - """Execute API tool calling workflow in streaming mode. - - Args: - request: Orchestration request. - context: Must contain "matched_endpoint" dict from APISemanticSearcher. - time_metric: Optional timing dict for step tracking. + ) -> Optional[OrchestrationResponse]: + """Execute one turn of the agentic loop and return a response. - Returns: - AsyncIterator yielding SSE-formatted strings, or None on failure. + Handles session creation (turn 1), resumption (turn 2-N), and all + AgenticLoopStatus outcomes. Returns None to signal a fallback only + when there is genuinely nothing to work with (no endpoint in context + and no active session). """ chat_id = request.chatId - endpoint = context.get("matched_endpoint") + session_store = self._get_session_store() + + # ── Try to resume an existing session ──────────────────────────── + session: Optional[APIToolSession] = None + if session_store is not None: + session = await session_store.get(chat_id) - if not endpoint: + if session is not None: + # Resume path — endpoint comes from persisted session + endpoint = session.selected_endpoint + if endpoint is None: + # Corrupt session: delete and fall back + logger.warning( + f"[{chat_id}] APIToolWorkflow: session has no endpoint — deleting" + ) + if session_store is not None: + await session_store.delete(chat_id) + return None + + logger.info( + f"[{chat_id}] APIToolWorkflow: resuming session " + f"(turn={session.turn_count}, endpoint={endpoint.get('name')!r})" + ) + else: + # New-session path — endpoint must come from classifier context + endpoint = context.get("matched_endpoint") + if not endpoint: + logger.warning( + f"[{chat_id}] APIToolWorkflow: no matched_endpoint in context " + f"and no active session — falling back" + ) + return None + + params_schema: List[Dict[str, Any]] = endpoint.get("params", []) + + # Fast path: no required params — nothing to collect, return immediately + if not self._required_params(params_schema): + logger.info( + f"[{chat_id}] APIToolWorkflow: endpoint {endpoint.get('name')!r} " + f"has no required params — fast path" + ) + return self._build_completed_response(chat_id, endpoint, {}) + + # Create a new session before running the first loop turn + if session_store is not None: + new_session = APIToolSession( + chat_id=chat_id, + state="collecting_params", + selected_endpoint=endpoint, + collected_params={}, + turn_count=0, + max_turns=5, + awaiting_continuation=False, + detected_language=getattr(request, "_detected_language", "en"), + ) + await session_store.save(new_session) + session = new_session + else: + # Redis unavailable — create an in-memory placeholder so the + # loop still runs (single-turn degradation: no persistence). + logger.warning( + f"[{chat_id}] APIToolWorkflow: Redis unavailable — " + f"running loop without session persistence" + ) + session = APIToolSession( + chat_id=chat_id, + state="collecting_params", + selected_endpoint=endpoint, + collected_params={}, + turn_count=0, + max_turns=5, + awaiting_continuation=False, + detected_language=getattr(request, "_detected_language", "en"), + ) + + # ── Run one loop turn ───────────────────────────────────────────── + if session_store is None: + # Without a store the loop cannot persist. + # so AgenticLoop doesn't crash. The loop will run but state is lost. logger.warning( - f"[{chat_id}] APIToolWorkflow streaming: no matched_endpoint — falling back" + f"[{chat_id}] APIToolWorkflow: session store unavailable — " + f"agentic loop running without persistence" ) - return None - if self.orchestration_service is None: - logger.error( - f"[{chat_id}] APIToolWorkflow streaming: orchestration_service not set" + loop = self._build_agentic_loop(session_store) # type: ignore[arg-type] + + result = await loop.run_turn( + chat_id=chat_id, + user_message=request.message, + # On the first turn of a new session (turn_count == 0) we pass an + # empty history so the extractor only looks at the current message. + # Full chat history may contain parameter values from a *previous* + # session that would be falsely re-used for this fresh request. + # On subsequent turns (turn_count > 0) the history is relevant — + # the user may refer back to something they said in this session. + conversation_history=( + [] + if session.turn_count == 0 + else [ + {"authorRole": item.authorRole, "message": item.message} + for item in (request.conversationHistory or []) + ] + ), + params_schema=endpoint.get("params", []), + collected_params=session.collected_params, + turn_count=session.turn_count, + max_turns=session.max_turns, + awaiting_continuation=session.awaiting_continuation, + session_language=session.detected_language, + ) + + # ── Handle result ───────────────────────────────────────────────── + if result.status == AgenticLoopStatus.COMPLETED: + logger.info( + f"[{chat_id}] APIToolWorkflow: all params collected " + f"(turns={result.turn_count}, params={list(result.collected_params.keys())})" + ) + if session_store is not None: + await session_store.delete(chat_id) + return self._build_completed_response( + chat_id, endpoint, result.collected_params ) - return None - name = endpoint.get("name", "unknown") - description = endpoint.get("description", "") - url = endpoint.get("url", "N/A") - confidence = endpoint.get("confidence", "medium") - cosine_score = endpoint.get("cosine_score", 0.0) + if result.status == AgenticLoopStatus.MAX_TURNS_REACHED: + logger.info( + f"[{chat_id}] APIToolWorkflow: max turns reached — deleting session" + ) + if session_store is not None: + await session_store.delete(chat_id) + # Return None to trigger fallback to the RAG workflow + return None + # NEEDS_INPUT or AWAITING_CONTINUATION_DECISION — session already saved + # by the loop; return the question to the user logger.info( - f"[{chat_id}] APIToolWorkflow streaming: matched endpoint={name!r} " - f"(confidence={confidence}, cosine={cosine_score:.4f})" + f"[{chat_id}] APIToolWorkflow: asking for more info " + f"(status={result.status.value}, turn={result.turn_count})" ) + return self._build_question_response(chat_id, result.clarifying_question) + + # ------------------------------------------------------------------ + # BaseWorkflow interface + # ------------------------------------------------------------------ + + async def execute_async( + self, + request: OrchestrationRequest, + context: Dict[str, Any], + time_metric: Optional[Dict[str, float]] = None, + ) -> Optional[OrchestrationResponse]: + return await self._run(request, context) + + async def execute_streaming( + self, + request: OrchestrationRequest, + context: Dict[str, Any], + time_metric: Optional[Dict[str, float]] = None, + ) -> Optional[AsyncIterator[str]]: + """Streaming mode — run the loop and wrap the response in SSE frames. - # TODO (Task 10): Replace with full agentic loop streaming - content = f"**{name}**: {description}\n\nURL: {url}" + Clarifying questions and the final params-collected response are both + short strings, so they are emitted as a single SSE frame + END marker. + Full token-by-token streaming of the final API response will be added + when the API caller is implemented. + """ + if self.orchestration_service is None: + logger.error( + f"[{request.chatId}] APIToolWorkflow streaming: orchestration_service not set" + ) + return None + + response = await self._run(request, context) + if response is None: + return None orchestration_service = self.orchestration_service + content = response.content async def _stream() -> AsyncIterator[str]: - yield orchestration_service.format_sse(chat_id, content) - yield orchestration_service.format_sse(chat_id, "END") + yield orchestration_service.format_sse(request.chatId, content) + yield orchestration_service.format_sse(request.chatId, "END") return _stream() diff --git a/src/utils/language_detector.py b/src/utils/language_detector.py index 8b95faa4..db79988a 100644 --- a/src/utils/language_detector.py +++ b/src/utils/language_detector.py @@ -70,7 +70,6 @@ def detect_language(text: str) -> LanguageCode: "võib", "olen", "oled", - "see", "seda", "jah", "või", diff --git a/tests/api_tool_eval/integration-results.json b/tests/api_tool_eval/integration-results.json new file mode 100644 index 00000000..d5e3192e --- /dev/null +++ b/tests/api_tool_eval/integration-results.json @@ -0,0 +1,191 @@ +[ + { + "name": "1 — Single-turn complete (vehicle tax with reg number)", + "passed": true, + "error": "", + "turns": [ + { + "turn": 1, + "message_sent": "Calculate vehicle tax for registration number 123ABC", + "response_content": "{\"status\": \"params_collected\", \"endpoint\": {\"name\": \"get_vehicle_tax_info\"}, \"collected_params\": {\"registrationNumber\": \"123ABC\"}}", + "passed": true, + "note": "collected_params={'registrationNumber': '123ABC'}" + } + ] + }, + { + "name": "2 — Multi-turn EN (public holidays, params across 2 turns)", + "passed": true, + "error": "", + "turns": [ + { + "turn": 1, + "message_sent": "What are the public holidays in Estonia this year?", + "response_content": "Could you please specify the start date and end date for the period you want to know about the public holidays in Estonia? For example, you can provide the beginning and ending dates in the format YYYY-MM-DD.", + "passed": true, + "note": "" + }, + { + "turn": 2, + "message_sent": "EE, from 2025-01-01 to 2025-12-31", + "response_content": "{\"status\": \"params_collected\", \"endpoint\": {\"name\": \"get_public_holidays\"}, \"collected_params\": {\"validFrom\": \"2025-01-01\", \"validTo\": \"2025-12-31\", \"countryIsoCode\": \"EE\"}}", + "passed": true, + "note": "collected_params={'validFrom': '2025-01-01', 'validTo': '2025-12-31', 'countryIsoCode': 'EE'}" + } + ] + }, + { + "name": "4 — No-params fast-path (parliament votings)", + "passed": true, + "error": "", + "turns": [ + { + "turn": 1, + "message_sent": "Show me the latest parliament voting records in Estonia", + "response_content": "{\"status\": \"params_collected\", \"endpoint\": {\"name\": \"get_parliament_votings\"}, \"collected_params\": {}}", + "passed": true, + "note": "endpoint={'name': 'get_parliament_votings'}, collected_params={}" + } + ] + }, + { + "name": "5 — Multi-turn (address search, 2 turns)", + "passed": true, + "error": "", + "turns": [ + { + "turn": 1, + "message_sent": "Search for an address in Estonia", + "response_content": "What is the address or place name you would like to search for in Estonia?", + "passed": true, + "note": "" + }, + { + "turn": 2, + "message_sent": "Viru 4, Tallinn", + "response_content": "{\"status\": \"params_collected\", \"endpoint\": {\"name\": \"search_address\"}, \"collected_params\": {\"address\": \"Viru 4, Tallinn\"}}", + "passed": true, + "note": "collected_params={'address': 'Viru 4, Tallinn'}" + } + ] + }, + { + "name": "6 — Multi-turn (electricity prices, datetime params)", + "passed": true, + "error": "", + "turns": [ + { + "turn": 1, + "message_sent": "What are the electricity market prices in Estonia?", + "response_content": "Could you please specify the start and end time period for which you would like to see the electricity market prices? Please provide both dates in ISO 8601 format (YYYY-MM-DDTHH:MM:SS).", + "passed": true, + "note": "" + }, + { + "turn": 2, + "message_sent": "From 2025-01-01T00:00:00 to 2025-01-07T23:59:59", + "response_content": "{\"status\": \"params_collected\", \"endpoint\": {\"name\": \"get_electricity_prices\"}, \"collected_params\": {\"start\": \"2025-01-01T00:00:00\", \"end\": \"2025-01-07T23:59:59\"}}", + "passed": true, + "note": "collected_params={'start': '2025-01-01T00:00:00', 'end': '2025-01-07T23:59:59'}" + } + ] + }, + { + "name": "7 — Session isolation (no param leak across flows)", + "passed": true, + "error": "", + "turns": [ + { + "turn": 1, + "message_sent": "Calculate vehicle tax for registration number 777XYZ", + "response_content": "{\"status\": \"params_collected\", \"endpoint\": {\"name\": \"get_vehicle_tax_info\"}, \"collected_params\": {\"registrationNumber\": \"777XYZ\"}}", + "passed": true, + "note": "" + }, + { + "turn": 2, + "message_sent": "What are the public holidays in Estonia this year?", + "response_content": "For which period would you like to see the public holidays? Please specify the start and end dates (in format YYYY-MM-DD).", + "passed": true, + "note": "Correctly started new session and asked for params" + } + ] + }, + { + "name": "8 — AWAITING_CONTINUATION_DECISION (yes → loop resumes)", + "passed": true, + "error": "", + "turns": [ + { + "turn": 1, + "message_sent": "I want to see public holidays", + "response_content": "Millise riigi ja millise ajavahemiku (algus- ja lõppkuupäev, kujul YYYY-MM-DD) riigipühasid soovite näha?", + "passed": true, + "note": "" + }, + { + "turn": 2, + "message_sent": "I'm not sure", + "response_content": "Millise riigi ja millise ajavahemiku (algus- ja lõppkuupäev, kujul YYYY-MM-DD) riigipühasid soovite näha?", + "passed": true, + "note": "" + }, + { + "turn": 3, + "message_sent": "I don't know", + "response_content": "Mul on vaja veel natuke lisateavet, kuid oleme selle kallal juba mõnda aega töötanud. Kas soovite jätkata ja vastata veel mõnele küsimusele, või eelistaksite peatuda ja saada üldise vastuse? (jah / ei)", + "passed": true, + "note": "Got continuation prompt" + }, + { + "turn": 4, + "message_sent": "yes", + "response_content": "Millise riigi kahetähelist ISO koodi (nt EE, LV) ning millist algus- ja lõppkuupäeva (kujul YYYY-MM-DD) soovite riigipühade nägemiseks kasutada?", + "passed": true, + "note": "Loop resumed correctly after 'yes'" + } + ] + }, + { + "name": "9 — MAX_TURNS_REACHED (loop exhausted, falls back to RAG)", + "passed": true, + "error": "", + "turns": [ + { + "turn": 1, + "message_sent": "I want to see public holidays", + "response_content": "Millise riigi ja kuupäevavahemiku (algus- ja lõppkuupäev formaadis YYYY-MM-DD) riigipühi soovite näha?", + "passed": true, + "note": "Still in loop" + }, + { + "turn": 2, + "message_sent": "hmm not sure", + "response_content": "Millise riigi ja kuupäevavahemiku (algus- ja lõppkuupäev formaadis YYYY-MM-DD) riigipühi soovite näha?", + "passed": true, + "note": "Still in loop" + }, + { + "turn": 3, + "message_sent": "I have no idea", + "response_content": "Mul on vaja veel natuke lisateavet, kuid oleme selle kallal juba mõnda aega töötanud. Kas soovite jätkata ja vastata veel mõnele küsimusele, või eelistaksite peatuda ja saada üldise vastuse? (jah / ei)", + "passed": true, + "note": "Still in loop" + }, + { + "turn": 4, + "message_sent": "yes", + "response_content": "Millise riigi ja millise kuupäevavahemiku (algus- ja lõppkuupäev formaadis YYYY-MM-DD) riigipühi soovite näha?", + "passed": true, + "note": "Still in loop" + }, + { + "turn": 5, + "message_sent": "I still don't know", + "response_content": "Millise riigi ja millise kuupäevavahemiku (algus- ja lõppkuupäev formaadis YYYY-MM-DD) riigipühi soovite näha?", + "passed": true, + "note": "Correctly fell back to RAG/OOD after max turns" + } + ] + } +] \ No newline at end of file diff --git a/tests/api_tool_eval/integration_test_agentic_loop.py b/tests/api_tool_eval/integration_test_agentic_loop.py new file mode 100644 index 00000000..f24eae75 --- /dev/null +++ b/tests/api_tool_eval/integration_test_agentic_loop.py @@ -0,0 +1,805 @@ +""" +Integration Test — Agentic Loop Multi-Turn Parameter Collection +============================================================== + +Tests the full end-to-end agentic loop via the /orchestrate endpoint. + +Scenarios covered: + 1. Single-turn complete — all params in first message (vehicle tax) + 2. Multi-turn (EN) — no params upfront, answered across 2 turns (public holidays) + 3. Multi-turn (ET) — same flow in Estonian (school holidays) + 4. No-params fast-path — endpoint with no required params (parliament votings) + 5. Address search — single required param, 2-turn + 6. Electricity prices — 2 required datetime params, 2-turn + 7. Session isolation — after completing one flow, a NEW query for the same chatId must NOT reuse old session values + 8. AWAITING_CONTINUATION_DECISION — hits continuation threshold, user says "yes", loop resumes + 9. MAX_TURNS_REACHED → loop falls back to RAG/OOD, does NOT return collected_params JSON + +Usage: + # Service running locally on port 8100 + uv run python tests/api_tool_eval/integration_test_agentic_loop.py + + # Against a different host/port + uv run python tests/api_tool_eval/integration_test_agentic_loop.py --url http://localhost:8100 + + # Keep going even after failures + uv run python tests/api_tool_eval/integration_test_agentic_loop.py --no-fail-fast + + # Save results to JSON + uv run python tests/api_tool_eval/integration_test_agentic_loop.py --output results-integration.json +""" + +import argparse +import json +import sys +import time +import uuid +from dataclasses import dataclass, field +from typing import Any, Dict, List, Optional, Tuple + +import requests + +# --------------------------------------------------------------------------- +# Config +# --------------------------------------------------------------------------- +DEFAULT_URL = "http://localhost:8100" +ORCHESTRATE_ENDPOINT = "/orchestrate" +ENVIRONMENT = "production" +AUTHOR_ID = "integration-test-user" +REQUEST_TIMEOUT = 30 # seconds per turn + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def make_chat_id(label: str) -> str: + """Unique chatId per test run so Redis sessions never collide across runs.""" + return f"integration-test-{label}-{uuid.uuid4().hex[:8]}" + + +def send_turn( + base_url: str, + chat_id: str, + message: str, + history: List[Dict[str, str]], + connection_id: Optional[str] = None, +) -> Dict[str, Any]: + """POST one turn to /orchestrate and return the parsed JSON response.""" + payload: Dict[str, Any] = { + "chatId": chat_id, + "message": message, + "authorId": AUTHOR_ID, + "conversationHistory": history, + "url": "integration-test", + "environment": ENVIRONMENT, + } + if connection_id: + payload["connection_id"] = connection_id + + resp = requests.post( + f"{base_url}{ORCHESTRATE_ENDPOINT}", + json=payload, + timeout=REQUEST_TIMEOUT, + ) + resp.raise_for_status() + return resp.json() + + +def append_to_history( + history: List[Dict[str, str]], + user_message: str, + bot_response: str, +) -> List[Dict[str, str]]: + """Return an updated conversation history list.""" + ts = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) + return history + [ + {"authorRole": "user", "message": user_message, "timestamp": ts}, + {"authorRole": "bot", "message": bot_response, "timestamp": ts}, + ] + + +def is_completed(content: str) -> bool: + """Return True if the response is a params-collected JSON payload.""" + try: + data = json.loads(content) + return "collected_params" in data and "endpoint" in data + except (json.JSONDecodeError, TypeError): + return False + + +def is_clarifying_question(content: str) -> bool: + """Return True if the response looks like a clarifying question (not JSON).""" + try: + json.loads(content) + return False # valid JSON → completed or error + except (json.JSONDecodeError, TypeError): + return bool(content.strip()) + + +# --------------------------------------------------------------------------- +# Result tracking +# --------------------------------------------------------------------------- + + +@dataclass +class TurnResult: + turn: int + message_sent: str + response_content: str + passed: bool + note: str = "" + + +@dataclass +class ScenarioResult: + name: str + passed: bool + turns: List[TurnResult] = field(default_factory=list) + error: str = "" + + +# --------------------------------------------------------------------------- +# Test scenarios +# --------------------------------------------------------------------------- + + +def scenario_1_single_turn_vehicle_tax(base_url: str) -> ScenarioResult: + """ + Scenario 1: Single-turn complete + -------------------------------- + User provides the required param (registrationNumber) in the first message. + Expected: response is immediately a completed JSON with collected_params. + """ + name = "1 — Single-turn complete (vehicle tax with reg number)" + chat_id = make_chat_id("s1") + history: List[Dict[str, str]] = [] + turns: List[TurnResult] = [] + + message = "Calculate vehicle tax for registration number 123ABC" + resp = send_turn(base_url, chat_id, message, history) + content = resp.get("content", "") + + passed = is_completed(content) + if passed: + data = json.loads(content) + collected = data.get("collected_params", {}) + passed = collected.get("registrationNumber") == "123ABC" + note = f"collected_params={collected}" + else: + note = f"Expected completed JSON, got: {content[:120]}" + + turns.append(TurnResult(1, message, content, passed, note)) + return ScenarioResult(name, passed, turns) + + +def scenario_2_multiturn_public_holidays_en(base_url: str) -> ScenarioResult: + """ + Scenario 2: Multi-turn — public holidays (English) + --------------------------------------------------- + Turn 1: vague query — bot asks for country + date range + Turn 2: user provides all params — bot returns completed JSON + """ + name = "2 — Multi-turn EN (public holidays, params across 2 turns)" + chat_id = make_chat_id("s2") + history: List[Dict[str, str]] = [] + turns: List[TurnResult] = [] + + # Turn 1 + msg1 = "What are the public holidays in Estonia this year?" + resp1 = send_turn(base_url, chat_id, msg1, history) + content1 = resp1.get("content", "") + + turn1_pass = is_clarifying_question(content1) + turns.append( + TurnResult( + 1, + msg1, + content1, + turn1_pass, + "Expected clarifying question" if not turn1_pass else "", + ) + ) + + if not turn1_pass: + return ScenarioResult( + name, False, turns, "Turn 1 did not return a clarifying question" + ) + + # Turn 2 + history = append_to_history(history, msg1, content1) + msg2 = "EE, from 2025-01-01 to 2025-12-31" + resp2 = send_turn(base_url, chat_id, msg2, history) + content2 = resp2.get("content", "") + + turn2_pass = is_completed(content2) + note2 = "" + if turn2_pass: + data = json.loads(content2) + collected = data.get("collected_params", {}) + expected_keys = {"countryIsoCode", "validFrom", "validTo"} + missing = expected_keys - collected.keys() + turn2_pass = not missing + note2 = ( + f"collected_params={collected}" + if not missing + else f"missing keys: {missing}" + ) + else: + note2 = f"Expected completed JSON, got: {content2[:120]}" + + turns.append(TurnResult(2, msg2, content2, turn2_pass, note2)) + overall = all(t.passed for t in turns) + return ScenarioResult(name, overall, turns) + + +def scenario_3_multiturn_school_holidays_et(base_url: str) -> ScenarioResult: + """ + Scenario 3: Multi-turn — school holidays (Estonian) + ---------------------------------------------------- + Turn 1: Estonian query, no params + Turn 2: provides date range in Estonian + """ + name = "3 — Multi-turn ET (school holidays in Estonian)" + chat_id = make_chat_id("s3") + history: List[Dict[str, str]] = [] + turns: List[TurnResult] = [] + + msg1 = "Millal on Eesti koolide koolivaheajad 2025. aastal?" + resp1 = send_turn(base_url, chat_id, msg1, history) + content1 = resp1.get("content", "") + + turn1_pass = is_clarifying_question(content1) + turns.append( + TurnResult( + 1, + msg1, + content1, + turn1_pass, + "Expected clarifying question" if not turn1_pass else "", + ) + ) + + if not turn1_pass: + return ScenarioResult( + name, False, turns, "Turn 1 did not return a clarifying question" + ) + + history = append_to_history(history, msg1, content1) + msg2 = "EE, alguskuupäev 2025-01-01, lõppkuupäev 2025-12-31" + resp2 = send_turn(base_url, chat_id, msg2, history) + content2 = resp2.get("content", "") + + turn2_pass = is_completed(content2) + note2 = "" + if turn2_pass: + data = json.loads(content2) + collected = data.get("collected_params", {}) + expected_keys = {"countryIsoCode", "validFrom", "validTo"} + missing = expected_keys - collected.keys() + turn2_pass = not missing + note2 = ( + f"collected_params={collected}" + if not missing + else f"missing keys: {missing}" + ) + else: + note2 = f"Expected completed JSON, got: {content2[:120]}" + + turns.append(TurnResult(2, msg2, content2, turn2_pass, note2)) + overall = all(t.passed for t in turns) + return ScenarioResult(name, overall, turns) + + +def scenario_4_no_params_fast_path(base_url: str) -> ScenarioResult: + """ + Scenario 4: No-params fast-path (parliament votings) + ----------------------------------------------------- + Endpoint has no required params → should return completed JSON on turn 1 + without asking any clarifying questions. + """ + name = "4 — No-params fast-path (parliament votings)" + chat_id = make_chat_id("s4") + history: List[Dict[str, str]] = [] + turns: List[TurnResult] = [] + + message = "Show me the latest parliament voting records in Estonia" + resp = send_turn(base_url, chat_id, message, history) + content = resp.get("content", "") + + passed = is_completed(content) + note = "" + if passed: + data = json.loads(content) + note = f"endpoint={data.get('endpoint')}, collected_params={data.get('collected_params')}" + else: + note = f"Expected fast-path completed JSON, got: {content[:120]}" + + turns.append(TurnResult(1, message, content, passed, note)) + return ScenarioResult(name, passed, turns) + + +def scenario_5_address_search(base_url: str) -> ScenarioResult: + """ + Scenario 5: Address search — single required param + --------------------------------------------------- + Turn 1: vague — "search for an address" → bot asks which address + Turn 2: user provides address → completed + """ + name = "5 — Multi-turn (address search, 2 turns)" + chat_id = make_chat_id("s5") + history: List[Dict[str, str]] = [] + turns: List[TurnResult] = [] + + msg1 = "Search for an address in Estonia" + resp1 = send_turn(base_url, chat_id, msg1, history) + content1 = resp1.get("content", "") + + turn1_pass = is_clarifying_question(content1) + turns.append( + TurnResult( + 1, + msg1, + content1, + turn1_pass, + "Expected clarifying question" if not turn1_pass else "", + ) + ) + + if not turn1_pass: + return ScenarioResult( + name, False, turns, "Turn 1 did not return a clarifying question" + ) + + history = append_to_history(history, msg1, content1) + msg2 = "Viru 4, Tallinn" + resp2 = send_turn(base_url, chat_id, msg2, history) + content2 = resp2.get("content", "") + + turn2_pass = is_completed(content2) + note2 = "" + if turn2_pass: + data = json.loads(content2) + collected = data.get("collected_params", {}) + turn2_pass = "address" in collected + note2 = f"collected_params={collected}" + else: + note2 = f"Expected completed JSON, got: {content2[:120]}" + + turns.append(TurnResult(2, msg2, content2, turn2_pass, note2)) + overall = all(t.passed for t in turns) + return ScenarioResult(name, overall, turns) + + +def scenario_6_electricity_prices(base_url: str) -> ScenarioResult: + """ + Scenario 6: Electricity prices — 2 required datetime params + ------------------------------------------------------------ + Turn 1: "What are the electricity prices?" → bot asks for start/end + Turn 2: user provides both datetimes → completed + """ + name = "6 — Multi-turn (electricity prices, datetime params)" + chat_id = make_chat_id("s6") + history: List[Dict[str, str]] = [] + turns: List[TurnResult] = [] + + msg1 = "What are the electricity market prices in Estonia?" + resp1 = send_turn(base_url, chat_id, msg1, history) + content1 = resp1.get("content", "") + + turn1_pass = is_clarifying_question(content1) + turns.append( + TurnResult( + 1, + msg1, + content1, + turn1_pass, + "Expected clarifying question" if not turn1_pass else "", + ) + ) + + if not turn1_pass: + return ScenarioResult( + name, False, turns, "Turn 1 did not return a clarifying question" + ) + + history = append_to_history(history, msg1, content1) + msg2 = "From 2025-01-01T00:00:00 to 2025-01-07T23:59:59" + resp2 = send_turn(base_url, chat_id, msg2, history) + content2 = resp2.get("content", "") + + turn2_pass = is_completed(content2) + note2 = "" + if turn2_pass: + data = json.loads(content2) + collected = data.get("collected_params", {}) + expected_keys = {"start", "end"} + missing = expected_keys - collected.keys() + turn2_pass = not missing + note2 = ( + f"collected_params={collected}" + if not missing + else f"missing keys: {missing}" + ) + else: + note2 = f"Expected completed JSON, got: {content2[:120]}" + + turns.append(TurnResult(2, msg2, content2, turn2_pass, note2)) + overall = all(t.passed for t in turns) + return ScenarioResult(name, overall, turns) + + +def scenario_7_session_isolation(base_url: str) -> ScenarioResult: + """ + Scenario 7: Session isolation after completion + ----------------------------------------------- + Uses the SAME chatId across two separate API tool flows to verify that + completing one flow does not leak params into the next query. + + Flow: + Turn 1: "Calculate vehicle tax for 777XYZ" → COMPLETED (session deleted) + Turn 2: "What are public holidays in Estonia?" → NEW session, asks for params + (MUST NOT immediately complete with registrationNumber=777XYZ) + """ + name = "7 — Session isolation (no param leak across flows)" + chat_id = make_chat_id("s7") + history: List[Dict[str, str]] = [] + turns: List[TurnResult] = [] + + # First flow — complete it + msg1 = "Calculate vehicle tax for registration number 777XYZ" + resp1 = send_turn(base_url, chat_id, msg1, history) + content1 = resp1.get("content", "") + + flow1_ok = is_completed(content1) + turns.append( + TurnResult( + 1, + msg1, + content1, + flow1_ok, + "First flow should complete immediately" if not flow1_ok else "", + ) + ) + + if not flow1_ok: + return ScenarioResult( + name, False, turns, "First flow did not complete — cannot test isolation" + ) + + # Second flow on the same chatId — must start fresh + history = append_to_history(history, msg1, content1) + msg2 = "What are the public holidays in Estonia this year?" + resp2 = send_turn(base_url, chat_id, msg2, history) + content2 = resp2.get("content", "") + + # It must NOT immediately return collected_params (that would mean param leak) + turn2_pass = is_clarifying_question(content2) + note2 = ( + "Correctly started new session and asked for params" + if turn2_pass + else f"BAD: returned completed JSON immediately (param leak?): {content2[:150]}" + ) + turns.append(TurnResult(2, msg2, content2, turn2_pass, note2)) + overall = all(t.passed for t in turns) + return ScenarioResult(name, overall, turns) + + +# --------------------------------------------------------------------------- +# Runner +# --------------------------------------------------------------------------- + + +def scenario_8_awaiting_continuation(base_url: str) -> ScenarioResult: + """ + Scenario 8: AWAITING_CONTINUATION_DECISION flow + ------------------------------------------------ + The loop hits CONTINUATION_TURN (3) without all params collected. + The bot must ask a yes/no "keep going?" question. + + We then answer "yes" to continue — the bot should resume asking for the + remaining params (not immediately complete and not fall back to RAG). + + Uses get_public_holidays which has 3 required params (countryIsoCode, + validFrom, validTo). We deliberately give unhelpful answers on turns 2 and 3 + to reach the continuation threshold. + + Turn flow (CONTINUATION_TURN=3, max_turns=5): + run_turn #1 (turn 0→1): opening question + run_turn #2 (turn 1→2): unhelpful reply → another clarifying question + run_turn #3 (turn 2→3): still unhelpful → AWAITING_CONTINUATION_DECISION + run_turn #4 (turn 3→4): user says "yes" → loop resumes, asks again + """ + name = "8 — AWAITING_CONTINUATION_DECISION (yes → loop resumes)" + chat_id = make_chat_id("s8") + history: List[Dict[str, str]] = [] + turns: List[TurnResult] = [] + + # Turn 1 — trigger the flow with no params + msg1 = "I want to see public holidays" + resp1 = send_turn(base_url, chat_id, msg1, history) + content1 = resp1.get("content", "") + t1_pass = is_clarifying_question(content1) + turns.append( + TurnResult( + 1, + msg1, + content1, + t1_pass, + "Expected opening clarifying question" if not t1_pass else "", + ) + ) + if not t1_pass: + return ScenarioResult( + name, False, turns, "Turn 1 did not return a clarifying question" + ) + + # Turn 2 — deliberately unhelpful + history = append_to_history(history, msg1, content1) + msg2 = "I'm not sure" + resp2 = send_turn(base_url, chat_id, msg2, history) + content2 = resp2.get("content", "") + t2_pass = is_clarifying_question(content2) and not is_completed(content2) + turns.append( + TurnResult( + 2, + msg2, + content2, + t2_pass, + "Expected follow-up clarifying question" if not t2_pass else "", + ) + ) + if not t2_pass: + return ScenarioResult( + name, False, turns, "Turn 2 did not return a clarifying question" + ) + + # Turn 3 — still unhelpful → should trigger continuation check + history = append_to_history(history, msg2, content2) + msg3 = "I don't know" + resp3 = send_turn(base_url, chat_id, msg3, history) + content3 = resp3.get("content", "") + # Continuation question contains "yes" or "no" and is not a completed JSON + is_continuation_prompt = not is_completed(content3) and ( + "yes" in content3.lower() + or "no" in content3.lower() + or "jah" in content3.lower() + ) + t3_pass = is_continuation_prompt + turns.append( + TurnResult( + 3, + msg3, + content3, + t3_pass, + "Expected yes/no continuation question" + if not t3_pass + else "Got continuation prompt", + ) + ) + if not t3_pass: + return ScenarioResult( + name, False, turns, "Turn 3 did not trigger continuation check" + ) + + # Turn 4 — user says "yes" → loop should resume with another clarifying question + history = append_to_history(history, msg3, content3) + msg4 = "yes" + resp4 = send_turn(base_url, chat_id, msg4, history) + content4 = resp4.get("content", "") + # After "yes" the bot must ask for params again, not complete + t4_pass = is_clarifying_question(content4) and not is_completed(content4) + note4 = ( + "Loop resumed correctly after 'yes'" + if t4_pass + else f"Expected resumed clarifying question, got: {content4[:150]}" + ) + turns.append(TurnResult(4, msg4, content4, t4_pass, note4)) + + overall = all(t.passed for t in turns) + return ScenarioResult(name, overall, turns) + + +def scenario_9_max_turns_reached(base_url: str) -> ScenarioResult: + """ + Scenario 9: MAX_TURNS_REACHED — loop falls back to RAG/OOD + ----------------------------------------------------------- + Keep giving unhelpful answers through the continuation "yes" and beyond + until max_turns (5) is exhausted. The final response must NOT be a + params-collected JSON — it should be a natural-language RAG/OOD answer. + + Turn flow (CONTINUATION_TURN=3, max_turns=5): + run_turn #1 (turn 0→1): opening question + run_turn #2 (turn 1→2): unhelpful → clarifying question + run_turn #3 (turn 2→3): unhelpful → AWAITING_CONTINUATION_DECISION + run_turn #4 (turn 3→4): "yes" → loop resumes, asks again + run_turn #5 (turn 4→5): unhelpful → MAX_TURNS_REACHED → fallback + """ + name = "9 — MAX_TURNS_REACHED (loop exhausted, falls back to RAG)" + chat_id = make_chat_id("s9") + history: List[Dict[str, str]] = [] + turns: List[TurnResult] = [] + + unhelpful_replies = [ + "I want to see public holidays", # turn 1 — trigger + "hmm not sure", # turn 2 — still missing + "I have no idea", # turn 3 — continuation check + "yes", # turn 4 — continue + "I still don't know", # turn 5 — max turns + ] + + last_content = "" + for i, msg in enumerate(unhelpful_replies, start=1): + resp = send_turn(base_url, chat_id, msg, history) + content = resp.get("content", "") + history = append_to_history(history, msg, content) + + if i < len(unhelpful_replies): + # Intermediate turns: should be asking questions or continuation prompt + intermediate_pass = not is_completed(content) + turns.append( + TurnResult( + i, + msg, + content, + intermediate_pass, + "Still in loop" + if intermediate_pass + else f"Unexpectedly completed at turn {i}", + ) + ) + if not intermediate_pass: + return ScenarioResult( + name, False, turns, f"Loop completed unexpectedly at turn {i}" + ) + else: + last_content = content + + # Final turn: must NOT be params-collected JSON (loop fell back to RAG/OOD) + final_pass = not is_completed(last_content) and bool(last_content.strip()) + note = ( + "Correctly fell back to RAG/OOD after max turns" + if final_pass + else f"BAD: got params-collected JSON after max turns: {last_content[:150]}" + ) + turns.append( + TurnResult( + len(unhelpful_replies), + unhelpful_replies[-1], + last_content, + final_pass, + note, + ) + ) + + overall = all(t.passed for t in turns) + return ScenarioResult(name, overall, turns) + + +SCENARIOS = [ + scenario_1_single_turn_vehicle_tax, + scenario_2_multiturn_public_holidays_en, + scenario_4_no_params_fast_path, + scenario_5_address_search, + scenario_6_electricity_prices, + scenario_7_session_isolation, + scenario_8_awaiting_continuation, + scenario_9_max_turns_reached, +] + + +def run_all( + base_url: str, fail_fast: bool = True +) -> Tuple[List[ScenarioResult], int, int]: + results: List[ScenarioResult] = [] + passed = 0 + failed = 0 + + print(f"\n{'=' * 70}") + print(f" Agentic Loop Integration Tests | {base_url}") + print(f"{'=' * 70}\n") + + for fn in SCENARIOS: + print(f"Running: {fn.__name__} ...", flush=True) + try: + result = fn(base_url) + except requests.exceptions.ConnectionError: + result = ScenarioResult( + fn.__name__, False, error="Connection refused — is the service running?" + ) + except requests.exceptions.Timeout: + result = ScenarioResult( + fn.__name__, False, error=f"Request timed out after {REQUEST_TIMEOUT}s" + ) + except Exception as exc: + result = ScenarioResult(fn.__name__, False, error=str(exc)) + + results.append(result) + status_icon = "✅" if result.passed else "❌" + print(f" {status_icon} {result.name}") + + for t in result.turns: + turn_icon = " ✓" if t.passed else " ✗" + print(f" {turn_icon} Turn {t.turn}: {t.message_sent[:60]!r}") + if t.note: + print(f" → {t.note}") + if not t.passed: + print(f" Response: {t.response_content[:200]}") + + if result.error: + print(f" ERROR: {result.error}") + + if result.passed: + passed += 1 + else: + failed += 1 + if fail_fast: + print("\n⚠ Stopping early (--no-fail-fast to continue)\n") + break + + print() + + print(f"{'=' * 70}") + print(f" Results: {passed} passed, {failed} failed / {len(results)} run") + print(f"{'=' * 70}\n") + + return results, passed, failed + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def main() -> None: + parser = argparse.ArgumentParser(description="Agentic loop integration tests") + parser.add_argument( + "--url", + default=DEFAULT_URL, + help=f"Base URL of the orchestration service (default: {DEFAULT_URL})", + ) + parser.add_argument( + "--no-fail-fast", + action="store_true", + help="Continue running all scenarios even after a failure", + ) + parser.add_argument( + "--output", + help="Optional path to save detailed JSON results", + ) + args = parser.parse_args() + + results, passed, failed = run_all( + base_url=args.url, + fail_fast=not args.no_fail_fast, + ) + + if args.output: + output_data = [ + { + "name": r.name, + "passed": r.passed, + "error": r.error, + "turns": [ + { + "turn": t.turn, + "message_sent": t.message_sent, + "response_content": t.response_content, + "passed": t.passed, + "note": t.note, + } + for t in r.turns + ], + } + for r in results + ] + with open(args.output, "w", encoding="utf-8") as f: + json.dump(output_data, f, ensure_ascii=False, indent=2) + print(f"Results saved to {args.output}\n") + + sys.exit(0 if failed == 0 else 1) + + +if __name__ == "__main__": + main()