Get update from llm-491 into llm-493

GUI/src/pages/TestProductionLLM/index.tsx

@@ -291,8 +291,6 @@ const TestProductionLLM: FC = () => {
   const clearChat = () => {
     setMessages([]);
     stopStreaming();
-    setInputMessage('');
-    setIsLoading(false);
     toast.open({
       type: 'info',
       title: t('testProductionLLM.chatClearedTitle'),
@@ -376,7 +374,7 @@ const TestProductionLLM: FC = () => {
               </div>
             ))}
 
-            {isLoading && (
+            {isLoading && (messages.length === 0 || messages[messages.length - 1].isUser) && (
               <div className="test-production-llm__message test-production-llm__message--bot">
                 <div className="test-production-llm__message-content">
                   <div className="test-production-llm__typing">

src/llm_orchestration_service.py

@@ -78,6 +78,8 @@
 # Initialize Loki logger for orchestration service
 logger = LokiLogger(service_name="llm-orchestration-service")
 
+REFERENCES_SECTION_HEADER = "\n\n**References:**\n"
+
 # Set of content strings that must NOT be persisted in conversation history.
 # Covers all multilingual error / OOS / guardrail-violation messages so that
 # failed or blocked exchanges are never written to Redis.
@@ -456,7 +458,6 @@ async def process_orchestration_request(
                     start_time = time.time()
                     classification = await self.tool_classifier.classify(
                         query=request.message,
-                        conversation_history=request.conversationHistory,
                         language=detected_language,
                         request=request,
                     )
@@ -738,7 +739,6 @@ async def stream_orchestration_response(
                         start_time = time.time()
                         classification = await self.tool_classifier.classify(
                             query=request.message,
-                            conversation_history=request.conversationHistory,
                             language=detected_language,
                             request=request,
                         )
@@ -1148,7 +1148,7 @@ async def bot_response_generator() -> AsyncIterator[str]:
                 # Send document references before END token
                 doc_references = self._extract_document_references(relevant_chunks)
                 if doc_references:
-                    refs_text = "\n\n**References:**\n" + "\n".join(
+                    refs_text = REFERENCES_SECTION_HEADER + "\n".join(
                         f"{i + 1}. [{ref.document_url}]({ref.document_url})"
                         for i, ref in enumerate(doc_references)
                     )
@@ -1185,7 +1185,7 @@ async def bot_response_generator() -> AsyncIterator[str]:
                 # Send document references before END token
                 doc_references = self._extract_document_references(relevant_chunks)
                 if doc_references:
-                    refs_text = "\n\n**References:**\n" + "\n".join(
+                    refs_text = REFERENCES_SECTION_HEADER + "\n".join(
                         f"{i + 1}. [{ref.document_url}]({ref.document_url})"
                         for i, ref in enumerate(doc_references)
                     )
@@ -2830,7 +2830,7 @@ def _generate_rag_response(
             doc_references = self._extract_document_references(relevant_chunks)
             content_with_refs = answer
             if doc_references:
-                refs_text = "\n\n**References:**\n" + "\n".join(
+                refs_text = REFERENCES_SECTION_HEADER + "\n".join(
                     f"{i + 1}. {ref.document_url}"
                     for i, ref in enumerate(doc_references)
                 )

src/llm_orchestrator_config/context_manager.py

@@ -53,7 +53,7 @@ def generate_context_with_caching(
             )
 
             # For now, call LLM directly (caching structure ready for future)
-            # TODO: Implement actual prompt caching when ready
+            # Implement actual prompt caching when ready
             response = self._call_llm_for_context(
                 prompt=full_prompt,
                 model=model_info["model"],

src/llm_orchestrator_config/llm_ochestrator_constants.py

@@ -26,7 +26,9 @@
 # Query validation messages - single generic message for all rejection types
 # (empty queries, special characters only, too short, repetitive characters)
 QUERY_VALIDATION_FAILED_MESSAGES = {
-    "et": "Palun esitage kehtiv küsimus või sõnum, et ma saaksin teid aidata."
+    "et": "Palun esitage kehtiv küsimus või sõnum, et ma saaksin teid aidata.",
+    "en": "Please provide a valid question or reply so I can assist you.",
+    "ru": "Пожалуйста, введите корректный вопрос или сообщение, чтобы я мог вам помочь.",
 }
 
 # Legacy constants for backward compatibility (English defaults)

src/tool_classifier/classifier.py

@@ -18,7 +18,6 @@
 from src.loki_logger import LokiLogger
 from llm_orchestrator_config.llm_manager import LLMManager
 from models.request_models import (
-    ConversationItem,
     OrchestrationRequest,
     OrchestrationResponse,
     TestOrchestrationResponse,
@@ -161,7 +160,6 @@ async def aclose(self) -> None:
     async def classify(
         self,
         query: str,
-        conversation_history: List[ConversationItem],
         language: str,
         request: Optional[OrchestrationRequest] = None,
     ) -> ClassificationResult:
@@ -179,7 +177,6 @@ async def classify(
 
         Args:
             query: User's query string
-            conversation_history: List of previous conversation messages
             language: Detected language code (e.g., 'en', 'et')
             request: Original orchestration request (needed for ATC search
                 which requires environment and connection_id for embedding).

src/tool_classifier/multi_response_formatter.py

@@ -495,7 +495,6 @@ def _build_results_block(
                         truncated
                         + "\n[NOTE: Combined results truncated due to total size limit]"
                     )
-                    total_bytes = _MAX_TOTAL_RESPONSE_BYTES
                 break
 
             sections.append(section)

src/tool_classifier/workflows/api_tool_workflow.py

@@ -7,6 +7,7 @@
     TYPE_CHECKING,
     Any,
     AsyncIterator,
+    Coroutine,
     Dict,
     List,
     Literal,
@@ -144,12 +145,28 @@ def __init__(
             if orchestration_service is not None
             else None
         )
+        self._background_tasks: set[asyncio.Task[None]] = set()
 
     # ------------------------------------------------------------------
     # Internal helpers
 
     # ------------------------------------------------------------------
 
+    def _create_background_task(self, coro: Coroutine[Any, Any, None]) -> None:
+        """Keep fire-and-forget tasks alive until they finish."""
+        task = asyncio.create_task(coro)
+        self._background_tasks.add(task)
+        task.add_done_callback(self._discard_background_task)
+
+    def _discard_background_task(self, task: asyncio.Task[None]) -> None:
+        """Remove a completed background task and log any uncaught exception."""
+        self._background_tasks.discard(task)
+        if task.cancelled():
+            return
+        exception = task.exception()
+        if exception is not None:
+            logger.warning(f"APIToolWorkflow: background task failed: {exception}")
+
     def _get_session_store(self) -> Optional[APIToolSessionStore]:
         """Return the session store from the orchestration service, or None."""
         if self.orchestration_service is None:
@@ -361,7 +378,7 @@ async def _write_l1_l2() -> None:
                             f"[{chat_id}] ATC cache: background write failed: {_exc}"
                         )
 
-                asyncio.create_task(_write_l1_l2())
+                self._create_background_task(_write_l1_l2())
             formatter = APIResponseFormatterModule(
                 custom_instructions=custom_instructions
             )
@@ -474,7 +491,7 @@ async def _write_multi_cache() -> None:
                         f"[{chat_id}] ATC cache: background multi-write failed: {_exc}"
                     )
 
-            asyncio.create_task(_write_multi_cache())
+            self._create_background_task(_write_multi_cache())
 
         api_results = [
             (
@@ -586,7 +603,7 @@ async def _write_multi_cache() -> None:
                         f"[{chat_id}] ATC cache: background multi-write failed: {_exc}"
                     )
 
-            asyncio.create_task(_write_multi_cache())
+            self._create_background_task(_write_multi_cache())
 
         api_results = [
             (
@@ -1442,7 +1459,7 @@ async def _write_l1_l2() -> None:
                             f"[{chat_id}] ATC cache: background write failed: {_exc}"
                         )
 
-                asyncio.create_task(_write_l1_l2())
+                self._create_background_task(_write_l1_l2())
             # Buffer all tokens first, then validate with output guardrails before
             # streaming to the client (validate-first approach).
             formatter = APIResponseFormatterModule(

src/tool_classifier/workflows/ood_workflow.py

@@ -25,13 +25,6 @@ class OODWorkflowExecutor(BaseWorkflow):
     - "Tell me a joke" (not government service)
     - Questions with no relevant knowledge
 
-    Implementation Status: SKELETON
-    Returns None (will implement to return OOD message)
-
-    TODO - Implementation (Simple):
-    - Return localized OUT_OF_SCOPE_MESSAGE
-    - Set questionOutOfLLMScope flag to True
-    - For streaming: chunk message and stream for UX consistency
     """
 
     def __init__(self) -> None:
@@ -83,7 +76,7 @@ async def execute_async(
             f"(not implemented - returning None for now)"
         )
 
-        # TODO: Implement OOD response logic here
+        # Implement OOD response logic here
         # For now, return None (will be implemented as simple message return)
         return None
 
@@ -132,6 +125,6 @@ async def stream_ood_message():
             f"(not implemented - returning None for now)"
         )
 
-        # TODO: Implement OOD streaming logic here
+        # Implement OOD streaming logic here
         # For now, return None (will be implemented as simple message streaming)
         return None

src/tool_classifier/workflows/service_workflow.py

@@ -45,6 +45,8 @@
 # Initialize Loki logger
 logger = LokiLogger(service_name="service-workflow")
 
+SERVICE_INTENT_DETECTION_METRIC = "service.intent_detection"
+
 
 class LLMServiceProtocol(Protocol):
     """Protocol defining interface for LLM service embedding operations."""
@@ -823,7 +825,7 @@ async def execute_async(
                     context=context,
                     costs_metric=costs_metric,
                 )
-                time_metric["service.intent_detection"] = time.time() - start_time
+                time_metric[SERVICE_INTENT_DETECTION_METRIC] = time.time() - start_time
 
                 if not context.get("service_data"):
                     context["service_id"] = matched.get("service_id")
@@ -845,7 +847,7 @@ async def execute_async(
                     context=context,
                     costs_metric=costs_metric,
                 )
-            time_metric["service.intent_detection"] = time.time() - start_time
+            time_metric[SERVICE_INTENT_DETECTION_METRIC] = time.time() - start_time
 
         else:
             start_time = time.time()
@@ -1005,7 +1007,7 @@ async def execute_streaming(
                     context=context,
                     costs_metric=costs_metric,
                 )
-                time_metric["service.intent_detection"] = time.time() - start_time
+                time_metric[SERVICE_INTENT_DETECTION_METRIC] = time.time() - start_time
 
                 if not context.get("service_data"):
                     context["service_id"] = matched.get("service_id")
@@ -1027,7 +1029,7 @@ async def execute_streaming(
                     context=context,
                     costs_metric=costs_metric,
                 )
-            time_metric["service.intent_detection"] = time.time() - start_time
+            time_metric[SERVICE_INTENT_DETECTION_METRIC] = time.time() - start_time
 
         else:
             start_time = time.time()

tests/test_tool_classifier.py

@@ -283,7 +283,6 @@ async def test_active_session_returns_api_tool_calling(self) -> None:
         ):
             result = await classifier.classify(
                 query="EE",
-                conversation_history=[],
                 language="en",
                 request=request,
             )
@@ -308,7 +307,6 @@ async def test_no_active_session_does_not_short_circuit(self) -> None:
         ):
             result = await classifier.classify(
                 query="some query",
-                conversation_history=[],
                 language="en",
                 request=request,
             )
@@ -345,7 +343,6 @@ async def test_different_endpoint_match_deletes_old_session(self) -> None:
         ):
             result = await classifier.classify(
                 query="What is the weather in Tallinn?",
-                conversation_history=[],
                 language="en",
                 request=request,
             )
@@ -381,7 +378,6 @@ async def test_api_tool_match_when_service_disabled(self) -> None:
         ):
             result = await classifier.classify(
                 query="public holidays",
-                conversation_history=[],
                 language="en",
                 request=_make_request("public holidays"),
             )
@@ -409,7 +405,6 @@ async def test_context_fallback_when_service_disabled_and_no_api_match(
         ):
             result = await classifier.classify(
                 query="tell me a joke",
-                conversation_history=[],
                 language="en",
                 request=_make_request("tell me a joke"),
             )
@@ -436,7 +431,6 @@ async def test_embedding_failure_falls_back_to_context(self) -> None:
         ):
             result = await classifier.classify(
                 query="public holidays",
-                conversation_history=[],
                 language="en",
             )
 
@@ -474,7 +468,6 @@ async def test_qdrant_timeout_falls_back_to_context(self) -> None:
             classifier.api_tool_searcher.search = AsyncMock(return_value=[])
             result = await classifier.classify(
                 query="public holidays",
-                conversation_history=[],
                 language="en",
             )