diff --git a/GUI/src/pages/TestProductionLLM/index.tsx b/GUI/src/pages/TestProductionLLM/index.tsx index 8bd8e86..8ba70ea 100644 --- a/GUI/src/pages/TestProductionLLM/index.tsx +++ b/GUI/src/pages/TestProductionLLM/index.tsx @@ -291,8 +291,6 @@ const TestProductionLLM: FC = () => { const clearChat = () => { setMessages([]); stopStreaming(); - setInputMessage(''); - setIsLoading(false); toast.open({ type: 'info', title: t('testProductionLLM.chatClearedTitle'), @@ -376,7 +374,7 @@ const TestProductionLLM: FC = () => { ))} - {isLoading && ( + {isLoading && (messages.length === 0 || messages[messages.length - 1].isUser) && (
diff --git a/src/llm_orchestration_service.py b/src/llm_orchestration_service.py index 4af9d7a..f20fc03 100644 --- a/src/llm_orchestration_service.py +++ b/src/llm_orchestration_service.py @@ -78,6 +78,8 @@ # Initialize Loki logger for orchestration service logger = LokiLogger(service_name="llm-orchestration-service") +REFERENCES_SECTION_HEADER = "\n\n**References:**\n" + # Set of content strings that must NOT be persisted in conversation history. # Covers all multilingual error / OOS / guardrail-violation messages so that # failed or blocked exchanges are never written to Redis. @@ -456,7 +458,6 @@ async def process_orchestration_request( start_time = time.time() classification = await self.tool_classifier.classify( query=request.message, - conversation_history=request.conversationHistory, language=detected_language, request=request, ) @@ -738,7 +739,6 @@ async def stream_orchestration_response( start_time = time.time() classification = await self.tool_classifier.classify( query=request.message, - conversation_history=request.conversationHistory, language=detected_language, request=request, ) @@ -1148,7 +1148,7 @@ async def bot_response_generator() -> AsyncIterator[str]: # Send document references before END token doc_references = self._extract_document_references(relevant_chunks) if doc_references: - refs_text = "\n\n**References:**\n" + "\n".join( + refs_text = REFERENCES_SECTION_HEADER + "\n".join( f"{i + 1}. [{ref.document_url}]({ref.document_url})" for i, ref in enumerate(doc_references) ) @@ -1185,7 +1185,7 @@ async def bot_response_generator() -> AsyncIterator[str]: # Send document references before END token doc_references = self._extract_document_references(relevant_chunks) if doc_references: - refs_text = "\n\n**References:**\n" + "\n".join( + refs_text = REFERENCES_SECTION_HEADER + "\n".join( f"{i + 1}. [{ref.document_url}]({ref.document_url})" for i, ref in enumerate(doc_references) ) @@ -2830,7 +2830,7 @@ def _generate_rag_response( doc_references = self._extract_document_references(relevant_chunks) content_with_refs = answer if doc_references: - refs_text = "\n\n**References:**\n" + "\n".join( + refs_text = REFERENCES_SECTION_HEADER + "\n".join( f"{i + 1}. {ref.document_url}" for i, ref in enumerate(doc_references) ) diff --git a/src/llm_orchestrator_config/context_manager.py b/src/llm_orchestrator_config/context_manager.py index 1b6146a..09efb40 100644 --- a/src/llm_orchestrator_config/context_manager.py +++ b/src/llm_orchestrator_config/context_manager.py @@ -53,7 +53,7 @@ def generate_context_with_caching( ) # For now, call LLM directly (caching structure ready for future) - # TODO: Implement actual prompt caching when ready + # Implement actual prompt caching when ready response = self._call_llm_for_context( prompt=full_prompt, model=model_info["model"], diff --git a/src/llm_orchestrator_config/llm_ochestrator_constants.py b/src/llm_orchestrator_config/llm_ochestrator_constants.py index 248752d..82bfceb 100644 --- a/src/llm_orchestrator_config/llm_ochestrator_constants.py +++ b/src/llm_orchestrator_config/llm_ochestrator_constants.py @@ -26,7 +26,9 @@ # Query validation messages - single generic message for all rejection types # (empty queries, special characters only, too short, repetitive characters) QUERY_VALIDATION_FAILED_MESSAGES = { - "et": "Palun esitage kehtiv küsimus või sõnum, et ma saaksin teid aidata." + "et": "Palun esitage kehtiv küsimus või sõnum, et ma saaksin teid aidata.", + "en": "Please provide a valid question or reply so I can assist you.", + "ru": "Пожалуйста, введите корректный вопрос или сообщение, чтобы я мог вам помочь.", } # Legacy constants for backward compatibility (English defaults) diff --git a/src/tool_classifier/classifier.py b/src/tool_classifier/classifier.py index 9414d82..9c28e68 100644 --- a/src/tool_classifier/classifier.py +++ b/src/tool_classifier/classifier.py @@ -18,7 +18,6 @@ from src.loki_logger import LokiLogger from llm_orchestrator_config.llm_manager import LLMManager from models.request_models import ( - ConversationItem, OrchestrationRequest, OrchestrationResponse, TestOrchestrationResponse, @@ -161,7 +160,6 @@ async def aclose(self) -> None: async def classify( self, query: str, - conversation_history: List[ConversationItem], language: str, request: Optional[OrchestrationRequest] = None, ) -> ClassificationResult: @@ -179,7 +177,6 @@ async def classify( Args: query: User's query string - conversation_history: List of previous conversation messages language: Detected language code (e.g., 'en', 'et') request: Original orchestration request (needed for ATC search which requires environment and connection_id for embedding). diff --git a/src/tool_classifier/multi_response_formatter.py b/src/tool_classifier/multi_response_formatter.py index 9ebcdef..a00d03e 100644 --- a/src/tool_classifier/multi_response_formatter.py +++ b/src/tool_classifier/multi_response_formatter.py @@ -495,7 +495,6 @@ def _build_results_block( truncated + "\n[NOTE: Combined results truncated due to total size limit]" ) - total_bytes = _MAX_TOTAL_RESPONSE_BYTES break sections.append(section) diff --git a/src/tool_classifier/workflows/api_tool_workflow.py b/src/tool_classifier/workflows/api_tool_workflow.py index d50cd69..b52c21f 100644 --- a/src/tool_classifier/workflows/api_tool_workflow.py +++ b/src/tool_classifier/workflows/api_tool_workflow.py @@ -7,6 +7,7 @@ TYPE_CHECKING, Any, AsyncIterator, + Coroutine, Dict, List, Literal, @@ -144,12 +145,28 @@ def __init__( if orchestration_service is not None else None ) + self._background_tasks: set[asyncio.Task[None]] = set() # ------------------------------------------------------------------ # Internal helpers # ------------------------------------------------------------------ + def _create_background_task(self, coro: Coroutine[Any, Any, None]) -> None: + """Keep fire-and-forget tasks alive until they finish.""" + task = asyncio.create_task(coro) + self._background_tasks.add(task) + task.add_done_callback(self._discard_background_task) + + def _discard_background_task(self, task: asyncio.Task[None]) -> None: + """Remove a completed background task and log any uncaught exception.""" + self._background_tasks.discard(task) + if task.cancelled(): + return + exception = task.exception() + if exception is not None: + logger.warning(f"APIToolWorkflow: background task failed: {exception}") + def _get_session_store(self) -> Optional[APIToolSessionStore]: """Return the session store from the orchestration service, or None.""" if self.orchestration_service is None: @@ -361,7 +378,7 @@ async def _write_l1_l2() -> None: f"[{chat_id}] ATC cache: background write failed: {_exc}" ) - asyncio.create_task(_write_l1_l2()) + self._create_background_task(_write_l1_l2()) formatter = APIResponseFormatterModule( custom_instructions=custom_instructions ) @@ -474,7 +491,7 @@ async def _write_multi_cache() -> None: f"[{chat_id}] ATC cache: background multi-write failed: {_exc}" ) - asyncio.create_task(_write_multi_cache()) + self._create_background_task(_write_multi_cache()) api_results = [ ( @@ -586,7 +603,7 @@ async def _write_multi_cache() -> None: f"[{chat_id}] ATC cache: background multi-write failed: {_exc}" ) - asyncio.create_task(_write_multi_cache()) + self._create_background_task(_write_multi_cache()) api_results = [ ( @@ -1442,7 +1459,7 @@ async def _write_l1_l2() -> None: f"[{chat_id}] ATC cache: background write failed: {_exc}" ) - asyncio.create_task(_write_l1_l2()) + self._create_background_task(_write_l1_l2()) # Buffer all tokens first, then validate with output guardrails before # streaming to the client (validate-first approach). formatter = APIResponseFormatterModule( diff --git a/src/tool_classifier/workflows/ood_workflow.py b/src/tool_classifier/workflows/ood_workflow.py index dcf1c86..4f4bab5 100644 --- a/src/tool_classifier/workflows/ood_workflow.py +++ b/src/tool_classifier/workflows/ood_workflow.py @@ -25,13 +25,6 @@ class OODWorkflowExecutor(BaseWorkflow): - "Tell me a joke" (not government service) - Questions with no relevant knowledge - Implementation Status: SKELETON - Returns None (will implement to return OOD message) - - TODO - Implementation (Simple): - - Return localized OUT_OF_SCOPE_MESSAGE - - Set questionOutOfLLMScope flag to True - - For streaming: chunk message and stream for UX consistency """ def __init__(self) -> None: @@ -83,7 +76,7 @@ async def execute_async( f"(not implemented - returning None for now)" ) - # TODO: Implement OOD response logic here + # Implement OOD response logic here # For now, return None (will be implemented as simple message return) return None @@ -132,6 +125,6 @@ async def stream_ood_message(): f"(not implemented - returning None for now)" ) - # TODO: Implement OOD streaming logic here + # Implement OOD streaming logic here # For now, return None (will be implemented as simple message streaming) return None diff --git a/src/tool_classifier/workflows/service_workflow.py b/src/tool_classifier/workflows/service_workflow.py index a249070..2b1f8f5 100644 --- a/src/tool_classifier/workflows/service_workflow.py +++ b/src/tool_classifier/workflows/service_workflow.py @@ -45,6 +45,8 @@ # Initialize Loki logger logger = LokiLogger(service_name="service-workflow") +SERVICE_INTENT_DETECTION_METRIC = "service.intent_detection" + class LLMServiceProtocol(Protocol): """Protocol defining interface for LLM service embedding operations.""" @@ -823,7 +825,7 @@ async def execute_async( context=context, costs_metric=costs_metric, ) - time_metric["service.intent_detection"] = time.time() - start_time + time_metric[SERVICE_INTENT_DETECTION_METRIC] = time.time() - start_time if not context.get("service_data"): context["service_id"] = matched.get("service_id") @@ -845,7 +847,7 @@ async def execute_async( context=context, costs_metric=costs_metric, ) - time_metric["service.intent_detection"] = time.time() - start_time + time_metric[SERVICE_INTENT_DETECTION_METRIC] = time.time() - start_time else: start_time = time.time() @@ -1005,7 +1007,7 @@ async def execute_streaming( context=context, costs_metric=costs_metric, ) - time_metric["service.intent_detection"] = time.time() - start_time + time_metric[SERVICE_INTENT_DETECTION_METRIC] = time.time() - start_time if not context.get("service_data"): context["service_id"] = matched.get("service_id") @@ -1027,7 +1029,7 @@ async def execute_streaming( context=context, costs_metric=costs_metric, ) - time_metric["service.intent_detection"] = time.time() - start_time + time_metric[SERVICE_INTENT_DETECTION_METRIC] = time.time() - start_time else: start_time = time.time() diff --git a/tests/test_tool_classifier.py b/tests/test_tool_classifier.py index 5bda698..f1d3edb 100644 --- a/tests/test_tool_classifier.py +++ b/tests/test_tool_classifier.py @@ -283,7 +283,6 @@ async def test_active_session_returns_api_tool_calling(self) -> None: ): result = await classifier.classify( query="EE", - conversation_history=[], language="en", request=request, ) @@ -308,7 +307,6 @@ async def test_no_active_session_does_not_short_circuit(self) -> None: ): result = await classifier.classify( query="some query", - conversation_history=[], language="en", request=request, ) @@ -345,7 +343,6 @@ async def test_different_endpoint_match_deletes_old_session(self) -> None: ): result = await classifier.classify( query="What is the weather in Tallinn?", - conversation_history=[], language="en", request=request, ) @@ -381,7 +378,6 @@ async def test_api_tool_match_when_service_disabled(self) -> None: ): result = await classifier.classify( query="public holidays", - conversation_history=[], language="en", request=_make_request("public holidays"), ) @@ -409,7 +405,6 @@ async def test_context_fallback_when_service_disabled_and_no_api_match( ): result = await classifier.classify( query="tell me a joke", - conversation_history=[], language="en", request=_make_request("tell me a joke"), ) @@ -436,7 +431,6 @@ async def test_embedding_failure_falls_back_to_context(self) -> None: ): result = await classifier.classify( query="public holidays", - conversation_history=[], language="en", ) @@ -474,7 +468,6 @@ async def test_qdrant_timeout_falls_back_to_context(self) -> None: classifier.api_tool_searcher.search = AsyncMock(return_value=[]) result = await classifier.classify( query="public holidays", - conversation_history=[], language="en", )