From 643031ee2a67275dcaf5c0e610daec80865d0257 Mon Sep 17 00:00:00 2001 From: Manuel Porras Ojeda Date: Mon, 8 Jun 2026 00:41:32 +0200 Subject: [PATCH 1/7] text analysis config and docs --- docs/ai_utils.rst | 84 +++++++++++++++++++++++++ toolium/utils/ai_utils/text_analysis.py | 9 +++ 2 files changed, 93 insertions(+) diff --git a/docs/ai_utils.rst b/docs/ai_utils.rst index 7bcd7931..1c7114cc 100644 --- a/docs/ai_utils.rst +++ b/docs/ai_utils.rst @@ -91,6 +91,90 @@ you have (direct OpenAI access or Azure OpenAI): AZURE_OPENAI_ENDPOINT= OPENAI_API_VERSION= +Text Criteria Analysis +---------------------- + +Text criteria analysis evaluates how well an input text matches a set of target characteristics +(e.g., tone, style, clarity, domain vocabulary, specific described content, etc) using an LLM. +Toolium provides utilities to both retrieve a structured analysis and assert quality thresholds. + +Usage +~~~~~ + +You can use functions from the `toolium.utils.ai_utils.text_analysis` module: + +* **get_text_criteria_analysis()**: returns a JSON string with an overall score and low-scored criteria. +* **assert_text_criteria()**: validates that the overall score is above a threshold and raises `AssertionError` otherwise. + +.. code-block:: python + + from toolium.utils.ai_utils.text_analysis import get_text_criteria_analysis, assert_text_criteria + import json + + input_text = "Hey team! Quick heads-up: deployment is done, all checks green." + text_criteria = [ + "professional tone", + "clear and concise message", + "it includes the status of the deployment" + ] + + # Get analysis (JSON string) + raw_analysis = get_text_criteria_analysis( + text_input=input_text, + text_criteria=text_criteria, + model_name="gpt-4o-mini", # optional + azure=False # True for Azure OpenAI + ) + + analysis = json.loads(raw_analysis) + print(analysis["overall_match"]) + print(analysis["features"]) # Only low-scored features (<= 0.2) + +.. code-block:: python + + from toolium.utils.ai_utils.text_analysis import assert_text_criteria + + # Assert text quality against a minimum score + assert_text_criteria( + text_input="This message should be short, formal and informative.", + text_criteria=["formal tone", "brevity", "informative content"], + threshold=0.75, + model_name="gpt-4o-mini", # optional + azure=False # set True for Azure OpenAI + ) + +Response Format +~~~~~~~~~~~~~~~ + +`get_text_criteria_analysis()` expects the LLM to return a JSON object with: + +.. code-block:: json + + { + "overall_match": 0.82, + "features": [ + { + "name": "formal tone", + "score": 0.15 + } + ] + } + +Where: + +- `overall_match`: float in `[0.0, 1.0]` +- `features`: list containing only low-scored criteria (score `<= 0.2`) + +Assertion Behavior +~~~~~~~~~~~~~~~~~~ + +`assert_text_criteria()`: + +- parses the JSON response returned by `get_text_criteria_analysis()` +- compares `overall_match` with the provided `threshold` +- raises `AssertionError` if `overall_match < threshold` +- logs low-scored features to help diagnose why validation failed + Text Readability ---------------- diff --git a/toolium/utils/ai_utils/text_analysis.py b/toolium/utils/ai_utils/text_analysis.py index 89900421..af7887be 100644 --- a/toolium/utils/ai_utils/text_analysis.py +++ b/toolium/utils/ai_utils/text_analysis.py @@ -19,6 +19,7 @@ import logging from toolium.utils.ai_utils.openai import openai_request +from toolium.driver_wrappers_pool import DriverWrappersPool logger = logging.getLogger(__name__) @@ -83,6 +84,14 @@ def get_text_criteria_analysis(text_input, text_criteria, model_name=None, azure """ # Build prompt using base prompt and target features system_message = build_system_message(text_criteria) + # Get config from config if azure is true and configuration exists. + config = DriverWrappersPool.get_default_wrapper().config + model_name = model_name or config.get_optional('AI', 'text_analysis_model', 'gpt-4o-mini') + if azure: + for key in ('azure_api_key', 'azure_endpoint', 'api_version', 'azure_deployment'): + value = config.get_optional('AI', key) + if value: + kwargs[key] = value response, _ = openai_request(system_message, text_input, model_name, azure, **kwargs) return response From d53a2bbe241fa644d0351f43f11248efdeed2521 Mon Sep 17 00:00:00 2001 From: Manuel Porras Ojeda Date: Mon, 8 Jun 2026 09:43:37 +0200 Subject: [PATCH 2/7] import order --- toolium/utils/ai_utils/text_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolium/utils/ai_utils/text_analysis.py b/toolium/utils/ai_utils/text_analysis.py index af7887be..03fad43e 100644 --- a/toolium/utils/ai_utils/text_analysis.py +++ b/toolium/utils/ai_utils/text_analysis.py @@ -18,8 +18,8 @@ import json import logging -from toolium.utils.ai_utils.openai import openai_request from toolium.driver_wrappers_pool import DriverWrappersPool +from toolium.utils.ai_utils.openai import openai_request logger = logging.getLogger(__name__) From dfc00a183f924da7dc73ee15df158264a9d07ffd Mon Sep 17 00:00:00 2001 From: Manuel Porras Ojeda Date: Mon, 8 Jun 2026 19:22:20 +0200 Subject: [PATCH 3/7] config to openai method --- docs/ai_utils.rst | 12 ++++++++++-- toolium/utils/ai_utils/openai.py | 5 +++++ toolium/utils/ai_utils/text_analysis.py | 9 --------- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/docs/ai_utils.rst b/docs/ai_utils.rst index 1c7114cc..fa651e8c 100644 --- a/docs/ai_utils.rst +++ b/docs/ai_utils.rst @@ -76,8 +76,8 @@ For SpaCy, you also need to download the language model, i.e. for small English python -m spacy download en_core_web_sm -For OpenAI LLM, you need to set up your configuration in environment variables, that it may depend on the type of access -you have (direct OpenAI access or Azure OpenAI): +For OpenAI LLM, you need to set up your configuration in environment variables or include them at toolium configuration, +that it may depend on the type of access you have (direct OpenAI access or Azure OpenAI): .. code-block:: bash @@ -91,6 +91,14 @@ you have (direct OpenAI access or Azure OpenAI): AZURE_OPENAI_ENDPOINT= OPENAI_API_VERSION= +.. code-block:: text +[AI] +text_similarity_method: azure_openai +sentence_transformers_model: all-mpnet-base-v2 +azure_endpoint: https://your-endpoint.azure.com +api_version: 2025-01-01-preview +azure_deployment: gpt-4o-mini + Text Criteria Analysis ---------------------- diff --git a/toolium/utils/ai_utils/openai.py b/toolium/utils/ai_utils/openai.py index 0974f16c..eb06b8fd 100644 --- a/toolium/utils/ai_utils/openai.py +++ b/toolium/utils/ai_utils/openai.py @@ -51,6 +51,11 @@ def openai_request(system_message, user_message, model_name=None, azure=False, * response_format = kwargs.pop('response_format') kwargs.pop('response_format', None) client = AzureOpenAI(**kwargs) if azure else OpenAI(**kwargs) + if azure: + for key in ('azure_api_key', 'azure_endpoint', 'api_version', 'azure_deployment'): + value = config.get_optional('AI', key) + if value: + kwargs[key] = value messages = [] if isinstance(system_message, list): for prompt in system_message: diff --git a/toolium/utils/ai_utils/text_analysis.py b/toolium/utils/ai_utils/text_analysis.py index 03fad43e..89900421 100644 --- a/toolium/utils/ai_utils/text_analysis.py +++ b/toolium/utils/ai_utils/text_analysis.py @@ -18,7 +18,6 @@ import json import logging -from toolium.driver_wrappers_pool import DriverWrappersPool from toolium.utils.ai_utils.openai import openai_request logger = logging.getLogger(__name__) @@ -84,14 +83,6 @@ def get_text_criteria_analysis(text_input, text_criteria, model_name=None, azure """ # Build prompt using base prompt and target features system_message = build_system_message(text_criteria) - # Get config from config if azure is true and configuration exists. - config = DriverWrappersPool.get_default_wrapper().config - model_name = model_name or config.get_optional('AI', 'text_analysis_model', 'gpt-4o-mini') - if azure: - for key in ('azure_api_key', 'azure_endpoint', 'api_version', 'azure_deployment'): - value = config.get_optional('AI', key) - if value: - kwargs[key] = value response, _ = openai_request(system_message, text_input, model_name, azure, **kwargs) return response From 63020d4466b8cc021122e4d20f638e4638e294db Mon Sep 17 00:00:00 2001 From: Manuel Porras Ojeda Date: Tue, 9 Jun 2026 09:48:53 +0200 Subject: [PATCH 4/7] use set default to allow overwrite config --- toolium/utils/ai_utils/openai.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/toolium/utils/ai_utils/openai.py b/toolium/utils/ai_utils/openai.py index eb06b8fd..2437367b 100644 --- a/toolium/utils/ai_utils/openai.py +++ b/toolium/utils/ai_utils/openai.py @@ -55,7 +55,12 @@ def openai_request(system_message, user_message, model_name=None, azure=False, * for key in ('azure_api_key', 'azure_endpoint', 'api_version', 'azure_deployment'): value = config.get_optional('AI', key) if value: - kwargs[key] = value + kwargs.setdefault(key, value) + else: + for key in ('openai_api_key', 'openai_endpoint', 'openai_version', 'openai_deployment'): + value = config.get_optional('AI', key) + if value: + kwargs.setdefault(key, value) messages = [] if isinstance(system_message, list): for prompt in system_message: From c0de81f73fa46f45bf7d490107ca8c572eae51af Mon Sep 17 00:00:00 2001 From: Manuel Porras Ojeda Date: Tue, 9 Jun 2026 10:27:53 +0200 Subject: [PATCH 5/7] fixes to openai defaults --- docs/ai_utils.rst | 16 ++++++++++++++-- toolium/utils/ai_utils/openai.py | 2 +- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/docs/ai_utils.rst b/docs/ai_utils.rst index fa651e8c..920fc8cf 100644 --- a/docs/ai_utils.rst +++ b/docs/ai_utils.rst @@ -91,14 +91,15 @@ that it may depend on the type of access you have (direct OpenAI access or Azure AZURE_OPENAI_ENDPOINT= OPENAI_API_VERSION= -.. code-block:: text +*[AI]* section:: + [AI] text_similarity_method: azure_openai -sentence_transformers_model: all-mpnet-base-v2 azure_endpoint: https://your-endpoint.azure.com api_version: 2025-01-01-preview azure_deployment: gpt-4o-mini + Text Criteria Analysis ---------------------- @@ -151,6 +152,17 @@ You can use functions from the `toolium.utils.ai_utils.text_analysis` module: azure=False # set True for Azure OpenAI ) + +Configuration +~~~~~~~~~~~~~ + +Default text analysis method can be set in the properties.cfg file with the property *text_analysis_method* in +*[AI]* section:: + + [AI] + text_analysis_method: openai # Options: 'openai', 'azure_openai' (default: azure_openai) + + Response Format ~~~~~~~~~~~~~~~ diff --git a/toolium/utils/ai_utils/openai.py b/toolium/utils/ai_utils/openai.py index 2437367b..d79e66f5 100644 --- a/toolium/utils/ai_utils/openai.py +++ b/toolium/utils/ai_utils/openai.py @@ -57,7 +57,7 @@ def openai_request(system_message, user_message, model_name=None, azure=False, * if value: kwargs.setdefault(key, value) else: - for key in ('openai_api_key', 'openai_endpoint', 'openai_version', 'openai_deployment'): + for key in ('openai_api_key', 'openai_model', 'openai_temperature'): value = config.get_optional('AI', key) if value: kwargs.setdefault(key, value) From 2b19eb718258f86194c63318b2946b188f3ad1bb Mon Sep 17 00:00:00 2001 From: Manuel Porras Ojeda Date: Tue, 9 Jun 2026 15:30:14 +0200 Subject: [PATCH 6/7] fix model to kwargs --- toolium/utils/ai_utils/openai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolium/utils/ai_utils/openai.py b/toolium/utils/ai_utils/openai.py index d79e66f5..3677e2d3 100644 --- a/toolium/utils/ai_utils/openai.py +++ b/toolium/utils/ai_utils/openai.py @@ -57,7 +57,7 @@ def openai_request(system_message, user_message, model_name=None, azure=False, * if value: kwargs.setdefault(key, value) else: - for key in ('openai_api_key', 'openai_model', 'openai_temperature'): + for key in ('openai_api_key', 'openai_temperature'): value = config.get_optional('AI', key) if value: kwargs.setdefault(key, value) From f4d3e51d785de7009bdc1d7d9ea4c6b2d6dd7c88 Mon Sep 17 00:00:00 2001 From: Manuel Porras Ojeda Date: Wed, 10 Jun 2026 00:40:23 +0200 Subject: [PATCH 7/7] Update docs/ai_utils.rst Co-authored-by: Pablo Guijarro --- docs/ai_utils.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ai_utils.rst b/docs/ai_utils.rst index 920fc8cf..b672e917 100644 --- a/docs/ai_utils.rst +++ b/docs/ai_utils.rst @@ -76,8 +76,8 @@ For SpaCy, you also need to download the language model, i.e. for small English python -m spacy download en_core_web_sm -For OpenAI LLM, you need to set up your configuration in environment variables or include them at toolium configuration, -that it may depend on the type of access you have (direct OpenAI access or Azure OpenAI): +For OpenAI LLM, you need to set up your configuration using environment variables or include it in the Toolium configuration. +The required parameters will vary depending on your access type (direct OpenAI or Azure OpenAI): .. code-block:: bash