From e255e22191df588023726f558ca92b4cccd6ed99 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Wed, 22 Jul 2026 07:39:49 -0400 Subject: [PATCH 01/25] improved tabular analysis --- application/single_app/config.py | 2 +- .../functions_simplechat_operations.py | 100 +- .../single_app/functions_tabular_csv_query.py | 204 +++ .../functions_tabular_generated_exports.py | 1274 +++++++++++++++-- application/single_app/route_backend_chats.py | 587 +++++++- .../plugin_invocation_logger.py | 9 + .../tabular_processing_plugin.py | 241 +++- .../static/js/chat/chat-messages.js | 117 +- ...ULAR_ROW_ORCHESTRATION_REMEDIATION_PLAN.md | 192 +++ docs/explanation/release_notes.md | 19 + .../test_assistant_table_csv_artifact.py | 6 +- ...st_tabular_background_generated_exports.py | 30 +- .../test_tabular_large_result_pagination.py | 90 +- .../test_tabular_row_orchestration_scale.py | 1224 ++++++++++++++++ scripts/Migration-AISearch.state.json | 145 ++ scripts/Migration-Cosmos.state.json | 460 ++++++ ...chat_background_generated_export_status.py | 196 ++- 17 files changed, 4755 insertions(+), 141 deletions(-) create mode 100644 application/single_app/functions_tabular_csv_query.py create mode 100644 docs/explanation/fixes/TABULAR_ROW_ORCHESTRATION_REMEDIATION_PLAN.md create mode 100644 functional_tests/test_tabular_row_orchestration_scale.py create mode 100644 scripts/Migration-AISearch.state.json create mode 100644 scripts/Migration-Cosmos.state.json diff --git a/application/single_app/config.py b/application/single_app/config.py index 4c9b91a3e..a182549ff 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -97,7 +97,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.250.059" +VERSION = "0.250.061" IS_DEVELOPMENT = is_development_env_enabled() SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax') diff --git a/application/single_app/functions_simplechat_operations.py b/application/single_app/functions_simplechat_operations.py index 10c17d70b..18293a07d 100644 --- a/application/single_app/functions_simplechat_operations.py +++ b/application/single_app/functions_simplechat_operations.py @@ -667,6 +667,66 @@ def upload_generated_analysis_artifact_for_user( ) +def upload_generated_analysis_artifact_stream_for_user( + current_user_id: str, + conversation_id: str, + file_name: str, + file_stream: Any, + file_size: int, + capability: str = "analysis", + output_format: str = "", + summary: str = "", + artifact_idempotency_key: str = "", +) -> Dict[str, Any]: + """Upload a bounded-memory generated artifact stream for an authorized user.""" + normalized_user_id = str(current_user_id or "").strip() + normalized_conversation_id = str(conversation_id or "").strip() + normalized_file_name = _normalize_generated_document_file_name(file_name) + normalized_capability = str(capability or "analysis").strip().lower() or "analysis" + normalized_output_format = str(output_format or "").strip().lower() or os.path.splitext(normalized_file_name)[1].lower().lstrip(".") + normalized_summary = str(summary or "").strip() + + if not normalized_user_id: + raise ValueError("current_user_id is required") + if not normalized_conversation_id: + raise ValueError("conversation_id is required") + if not hasattr(file_stream, "read") or not hasattr(file_stream, "seek"): + raise ValueError("file_stream must be seekable and readable") + if not allowed_file(normalized_file_name): + raise ValueError("Generated file type is not supported") + + normalized_file_size = max(0, int(file_size or 0)) + if normalized_file_size <= 0: + raise ValueError("Generated artifact is empty") + + settings = get_settings() + max_artifact_size_mb = settings.get("max_generated_chat_artifact_size_mb", 500) + try: + max_artifact_size_mb = max(1, int(max_artifact_size_mb)) + except (TypeError, ValueError): + max_artifact_size_mb = 500 + + max_artifact_size_bytes = max_artifact_size_mb * 1024 * 1024 + if normalized_file_size > max_artifact_size_bytes: + raise ValueError( + f"Generated artifact exceeds the {max_artifact_size_mb} MB size limit" + ) + + file_stream.seek(0) + return _upload_generated_chat_artifact_for_current_user( + current_user_id=normalized_user_id, + conversation_id=normalized_conversation_id, + normalized_file_name=normalized_file_name, + file_content_bytes=file_stream, + artifact_metadata={ + "capability": normalized_capability, + "output_format": normalized_output_format, + "summary": normalized_summary, + }, + artifact_idempotency_key=artifact_idempotency_key, + ) + + def delete_blob_backed_chat_message_files(messages: Iterable[Dict[str, Any]]) -> int: """Delete blob-backed chat files referenced by the provided message documents.""" blob_service_client = CLIENTS.get("storage_account_office_docs_client") @@ -1743,6 +1803,7 @@ def _upload_generated_chat_artifact_for_current_user( normalized_file_name: str, file_content_bytes: bytes, artifact_metadata: Optional[Dict[str, Any]] = None, + artifact_idempotency_key: str = "", ) -> Dict[str, Any]: try: conversation_item = cosmos_conversations_container.read_item( @@ -1759,7 +1820,15 @@ def _upload_generated_chat_artifact_for_current_user( if not blob_service_client: raise RuntimeError("Blob storage client not available") - artifact_message_id = f"{conversation_id}_generated_file_{uuid.uuid4().hex}" + normalized_idempotency_key = str(artifact_idempotency_key or "").strip() + if normalized_idempotency_key: + artifact_suffix = uuid.uuid5( + uuid.NAMESPACE_URL, + f"simplechat-generated-artifact:{conversation_id}:{normalized_idempotency_key}", + ).hex + else: + artifact_suffix = uuid.uuid4().hex + artifact_message_id = f"{conversation_id}_generated_file_{artifact_suffix}" blob_path = ( f"{current_user_id}/{conversation_id}/generated/" f"{artifact_message_id}/{normalized_file_name}" @@ -1768,6 +1837,33 @@ def _upload_generated_chat_artifact_for_current_user( container=storage_account_personal_chat_container_name, blob=blob_path, ) + if normalized_idempotency_key: + try: + existing_message = cosmos_messages_container.read_item( + item=artifact_message_id, + partition_key=conversation_id, + ) + except CosmosResourceNotFoundError: + existing_message = None + if ( + isinstance(existing_message, dict) + and existing_message.get("role") == "file" + and existing_message.get("filename") == normalized_file_name + and existing_message.get("blob_path") == blob_path + and blob_client.exists() + ): + existing_metadata = existing_message.get("metadata") or {} + return { + "message": { + "id": artifact_message_id, + "file_name": normalized_file_name, + "blob_container": storage_account_personal_chat_container_name, + "blob_path": blob_path, + "capability": existing_metadata.get("generated_artifact_capability") or "analysis", + "output_format": existing_metadata.get("generated_artifact_output_format") or "", + }, + "conversation_id": conversation_id, + } blob_client.upload_blob( file_content_bytes, overwrite=True, @@ -1775,6 +1871,7 @@ def _upload_generated_chat_artifact_for_current_user( "conversation_id": conversation_id, "user_id": current_user_id, "generated_artifact": "true", + "idempotent_artifact": str(bool(normalized_idempotency_key)).lower(), }, ) @@ -1804,6 +1901,7 @@ def _upload_generated_chat_artifact_for_current_user( "generated_artifact_capability": artifact_capability, "generated_artifact_output_format": artifact_output_format, "generated_artifact_summary": artifact_summary, + "generated_artifact_idempotency_key": normalized_idempotency_key or None, "thread_info": { "thread_id": current_thread_id, "previous_thread_id": previous_thread_id, diff --git a/application/single_app/functions_tabular_csv_query.py b/application/single_app/functions_tabular_csv_query.py new file mode 100644 index 000000000..a8915fb4d --- /dev/null +++ b/application/single_app/functions_tabular_csv_query.py @@ -0,0 +1,204 @@ +# functions_tabular_csv_query.py +"""Shared bounded CSV query evaluation for foreground tools and durable exports.""" + +import ast + +import pandas + + +TABULAR_ROW_LOCAL_QUERY_AST_NODES = ( + ast.Expression, + ast.BoolOp, + ast.BinOp, + ast.UnaryOp, + ast.Compare, + ast.Name, + ast.Load, + ast.Constant, + ast.List, + ast.Tuple, + ast.Set, + ast.And, + ast.Or, + ast.Not, + ast.Eq, + ast.NotEq, + ast.Lt, + ast.LtE, + ast.Gt, + ast.GtE, + ast.In, + ast.NotIn, + ast.Is, + ast.IsNot, + ast.Add, + ast.Sub, + ast.Mult, + ast.Div, + ast.FloorDiv, + ast.Mod, + ast.Pow, + ast.BitAnd, + ast.BitOr, + ast.BitXor, + ast.USub, + ast.UAdd, + ast.Invert, +) + + +def _replace_backtick_column_references(expression): + """Replace pandas backtick column labels with parseable placeholder names.""" + output = [] + placeholder_index = 0 + character_index = 0 + active_quote = None + escaped = False + while character_index < len(expression): + character = expression[character_index] + if escaped: + output.append(character) + escaped = False + character_index += 1 + continue + if character == '\\' and active_quote: + output.append(character) + escaped = True + character_index += 1 + continue + if active_quote: + output.append(character) + if character == active_quote: + active_quote = None + character_index += 1 + continue + if character in {'\'', '"'}: + active_quote = character + output.append(character) + character_index += 1 + continue + if character != '`': + output.append(character) + character_index += 1 + continue + + closing_index = expression.find('`', character_index + 1) + if closing_index < 0 or not expression[character_index + 1:closing_index].strip(): + raise ValueError('Source query contains an invalid backtick column reference') + output.append(f'__simplechat_column_{placeholder_index}') + placeholder_index += 1 + character_index = closing_index + 1 + + return ''.join(output) + + +def validate_tabular_csv_query_expression(query_expression): + """Return a row-local expression or reject operations that change across chunks.""" + normalized_expression = str(query_expression or '').strip() + if not normalized_expression: + raise ValueError('Source query expression is required') + if '@' in normalized_expression: + raise ValueError( + 'Source query variables cannot be replayed in bounded chunks' + ) + + parseable_expression = _replace_backtick_column_references(normalized_expression) + try: + parsed_expression = ast.parse(parseable_expression, mode='eval') + except SyntaxError as exc: + raise ValueError('Source query is not a valid row-local expression') from exc + + unsupported_nodes = [ + node.__class__.__name__ + for node in ast.walk(parsed_expression) + if not isinstance(node, TABULAR_ROW_LOCAL_QUERY_AST_NODES) + ] + if unsupported_nodes: + raise ValueError( + 'Source query uses an operation that cannot be replayed equivalently in bounded chunks: ' + f'{unsupported_nodes[0]}' + ) + for node in ast.walk(parsed_expression): + if isinstance(node, ast.Name) and node.id.startswith('__') and not node.id.startswith('__simplechat_column_'): + raise ValueError('Source query contains an unsupported private identifier') + return normalized_expression + + +def detect_tabular_csv_numeric_columns(csv_stream, source_chunk_rows, tabular_plugin): + """Find columns that pandas can convert to numeric across every bounded chunk.""" + numeric_columns = None + csv_stream.seek(0) + for source_chunk in pandas.read_csv( + csv_stream, + keep_default_na=False, + dtype=str, + chunksize=max(1, int(source_chunk_rows or 1)), + ): + source_chunk = tabular_plugin._normalize_dataframe_columns(source_chunk) + if numeric_columns is None: + numeric_columns = set(source_chunk.columns) + for column_name in list(numeric_columns): + try: + pandas.to_numeric(source_chunk[column_name]) + except (TypeError, ValueError): + numeric_columns.discard(column_name) + csv_stream.seek(0) + return numeric_columns or set() + + +def iter_tabular_csv_query_rows( + csv_stream, + query_expression, + return_columns, + source_chunk_rows, + tabular_plugin, + start_source_row=0, + replay_stats=None, +): + """Yield physical source row numbers and query-matched records in bounded chunks.""" + source_chunk_rows = max(1, int(source_chunk_rows or 1)) + start_source_row = max(0, int(start_source_row or 0)) + normalized_query_expression = validate_tabular_csv_query_expression(query_expression) + numeric_columns = detect_tabular_csv_numeric_columns( + csv_stream, + source_chunk_rows, + tabular_plugin, + ) + parsed_return_columns = tabular_plugin._parse_optional_column_list_argument(return_columns) + + csv_stream.seek(0) + read_options = { + 'keep_default_na': False, + 'dtype': str, + 'chunksize': source_chunk_rows, + } + if start_source_row: + read_options['skiprows'] = lambda row_index: 0 < row_index <= start_source_row + + source_row_offset = start_source_row + for source_chunk in pandas.read_csv(csv_stream, **read_options): + source_chunk = tabular_plugin._normalize_dataframe_columns(source_chunk) + source_chunk.index = range(source_row_offset, source_row_offset + len(source_chunk)) + source_row_offset += len(source_chunk) + for column_name in numeric_columns: + if column_name in source_chunk.columns: + source_chunk[column_name] = pandas.to_numeric(source_chunk[column_name]) + + filtered_chunk, used_reviewer_style_fallback = tabular_plugin._apply_query_expression_with_fallback( + source_chunk, + query_expression=normalized_query_expression, + normalize_match=False, + ) + if isinstance(replay_stats, dict) and used_reviewer_style_fallback: + replay_stats['used_reviewer_style_fallback'] = True + selected_columns = [ + column_name + for column_name in (parsed_return_columns or list(filtered_chunk.columns)) + if column_name in filtered_chunk.columns + ] + output_records = tabular_plugin._build_row_output_records( + filtered_chunk, + selected_columns, + ) + for source_row_index, output_record in zip(filtered_chunk.index, output_records): + yield int(source_row_index) + 1, output_record diff --git a/application/single_app/functions_tabular_generated_exports.py b/application/single_app/functions_tabular_generated_exports.py index 6d27ef76d..150c3701b 100644 --- a/application/single_app/functions_tabular_generated_exports.py +++ b/application/single_app/functions_tabular_generated_exports.py @@ -2,35 +2,51 @@ """Durable background runs for large tabular generated exports.""" import asyncio +from collections import Counter import csv import io import json import logging +import math import os import re import socket +import tempfile import time import uuid from datetime import datetime, timedelta, timezone from azure.core import MatchConditions +from azure.core.exceptions import ResourceExistsError from azure.cosmos.exceptions import CosmosResourceNotFoundError from flask import current_app, has_app_context from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.azure_chat_prompt_execution_settings import AzureChatPromptExecutionSettings from semantic_kernel.contents.chat_history import ChatHistory as SKChatHistory +from semantic_kernel_plugins.tabular_processing_plugin import TabularProcessingPlugin from config import ( CLIENTS, + cosmos_conversations_container, cosmos_tabular_export_runs_container, + storage_account_group_documents_container_name, storage_account_personal_chat_container_name, + storage_account_public_documents_container_name, + storage_account_user_documents_container_name, ) from functions_appinsights import log_event +from functions_tabular_csv_query import ( + iter_tabular_csv_query_rows, + validate_tabular_csv_query_expression, +) +from functions_group import assert_group_role from functions_model_endpoint_runtime import build_semantic_kernel_chat_service_for_model +from functions_public_workspaces import get_user_visible_public_workspace_ids_from_settings from functions_settings import get_settings -from functions_simplechat_operations import upload_generated_analysis_artifact_for_user +from functions_simplechat_operations import upload_generated_analysis_artifact_stream_for_user TABULAR_EXPORT_RUN_TYPE = 'tabular_generated_output_run' +TABULAR_EXPORT_CONTRACT_VERSION = 2 TABULAR_EXPORT_STATUS_QUEUED = 'queued' TABULAR_EXPORT_STATUS_RUNNING = 'running' TABULAR_EXPORT_STATUS_COMPLETED = 'completed' @@ -51,6 +67,13 @@ TABULAR_EXPORT_DEFAULT_MAX_TRANSIENT_FAILURES = 20 TABULAR_EXPORT_DEFAULT_BATCH_CONCURRENCY = 2 TABULAR_EXPORT_MAX_BATCH_CONCURRENCY = 5 +TABULAR_EXPORT_FINAL_SPOOL_MAX_MEMORY_BYTES = 1024 * 1024 +TABULAR_EXPORT_DEFAULT_SOURCE_CHUNK_ROWS = 1000 +TABULAR_EXPORT_DEFAULT_SOURCE_BATCH_ROWS = 50 +TABULAR_EXPORT_DEFAULT_SOURCE_BATCH_CHARS = 60000 +TABULAR_EXPORT_SUMMARY_MAX_FIELDS = 25 +TABULAR_EXPORT_SUMMARY_MAX_VALUES_PER_FIELD = 5 +TABULAR_EXPORT_SUMMARY_AGGREGATE_MAX_VALUES = 25 TABULAR_EXPORT_PROGRESS_LOG_INTERVAL_SECONDS = 30 TABULAR_EXPORT_SCHEDULER_STATUSES = ( TABULAR_EXPORT_STATUS_QUEUED, @@ -87,6 +110,19 @@ 'worker exiting', 'worker restart', ) +TABULAR_EXPORT_INPUT_ROW_NUMBER_FIELD = '__simplechat_source_row_number' +TABULAR_EXPORT_INPUT_ROW_IDENTITY_FIELD = '__simplechat_source_row_identity' +TABULAR_EXPORT_INPUT_ROW_TOKEN_FIELD = '__simplechat_source_row_token' +TABULAR_EXPORT_OUTPUT_ROW_NUMBER_FIELD = 'source_row_number' +TABULAR_EXPORT_OUTPUT_ROW_IDENTITY_FIELD = 'source_row_identity' + + +class TabularExportCanceledError(RuntimeError): + """Raised when a durable export is canceled between checkpoints.""" + + +class TabularExportLeaseLostError(RuntimeError): + """Raised when a stale worker no longer owns the durable run claim.""" def _now_utc(): @@ -205,6 +241,262 @@ def _serialize_generated_output_value(value): return str(value) +def _normalize_source_identity_label(value): + return re.sub(r'[^a-z0-9]+', '', str(value or '').strip().casefold()) + + +def _select_source_row_identity(row, source_row_number): + if not isinstance(row, dict): + return str(source_row_number) + + identity_priorities = ( + 'sourceidentity', + 'sourceid', + 'caseid', + 'recordid', + 'rowid', + 'commentid', + 'submissionid', + 'id', + ) + normalized_values = {} + for field_name, field_value in row.items(): + if field_name in { + TABULAR_EXPORT_INPUT_ROW_NUMBER_FIELD, + TABULAR_EXPORT_INPUT_ROW_IDENTITY_FIELD, + TABULAR_EXPORT_INPUT_ROW_TOKEN_FIELD, + }: + continue + normalized_label = _normalize_source_identity_label(field_name) + if normalized_label and field_value not in (None, '', [], {}): + normalized_values.setdefault(normalized_label, field_value) + + for identity_label in identity_priorities: + identity_value = normalized_values.get(identity_label) + if identity_value not in (None, '', [], {}): + return str(identity_value) + + for identity_label, identity_value in normalized_values.items(): + if identity_label.endswith('id') and not isinstance(identity_value, (dict, list, tuple, set)): + return str(identity_value) + + return str(source_row_number) + + +def _prepare_tabular_source_rows(rows, start_row=0, token_namespace=''): + try: + normalized_start_row = max(0, int(start_row or 0)) + except (TypeError, ValueError): + normalized_start_row = 0 + + prepared_rows = [] + for row_offset, row in enumerate(rows or []): + source_row_number = normalized_start_row + row_offset + 1 + prepared_row = dict(row) if isinstance(row, dict) else {'value': row} + prepared_row[TABULAR_EXPORT_INPUT_ROW_NUMBER_FIELD] = source_row_number + prepared_row[TABULAR_EXPORT_INPUT_ROW_IDENTITY_FIELD] = _select_source_row_identity( + prepared_row, + source_row_number, + ) + token_seed = ( + f'simplechat-tabular-row:{token_namespace}:{source_row_number}:' + f'{prepared_row[TABULAR_EXPORT_INPUT_ROW_IDENTITY_FIELD]}' + ) + prepared_row[TABULAR_EXPORT_INPUT_ROW_TOKEN_FIELD] = uuid.uuid5( + uuid.NAMESPACE_URL, + token_seed, + ).hex + prepared_rows.append(prepared_row) + return prepared_rows + + +def _normalize_generated_batch_entries( + source_rows, + generated_entries, + expected_output_schema=None, + require_source_token=True, +): + source_rows = list(source_rows or []) + generated_entries = list(generated_entries or []) + if len(source_rows) != len(generated_entries): + raise ValueError( + f'Generated row count {len(generated_entries)} does not match source row count {len(source_rows)}' + ) + + normalized_entries = [] + for row_index, (source_row, generated_entry) in enumerate(zip(source_rows, generated_entries), start=1): + if not isinstance(source_row, dict): + raise ValueError(f'Source row {row_index} is not an object') + if not isinstance(generated_entry, dict): + raise ValueError(f'Generated row {row_index} is not an object') + + source_row_number = source_row.get(TABULAR_EXPORT_INPUT_ROW_NUMBER_FIELD) + source_row_identity = source_row.get(TABULAR_EXPORT_INPUT_ROW_IDENTITY_FIELD) + source_row_token = str(source_row.get(TABULAR_EXPORT_INPUT_ROW_TOKEN_FIELD) or '').strip() + if source_row_number in (None, '') or source_row_identity in (None, ''): + raise ValueError(f'Source identity is missing for generated row {row_index}') + generated_row_token = str( + generated_entry.get(TABULAR_EXPORT_INPUT_ROW_TOKEN_FIELD) or '' + ).strip() + if require_source_token and ( + not source_row_token + or generated_row_token != source_row_token + ): + raise ValueError( + f'Generated source row token mismatch at row {row_index}' + ) + + normalized_entry = { + TABULAR_EXPORT_OUTPUT_ROW_NUMBER_FIELD: source_row_number, + TABULAR_EXPORT_OUTPUT_ROW_IDENTITY_FIELD: str(source_row_identity), + } + normalized_entry.update({ + str(field_name): field_value + for field_name, field_value in generated_entry.items() + if str(field_name) not in { + TABULAR_EXPORT_INPUT_ROW_NUMBER_FIELD, + TABULAR_EXPORT_INPUT_ROW_IDENTITY_FIELD, + TABULAR_EXPORT_INPUT_ROW_TOKEN_FIELD, + TABULAR_EXPORT_OUTPUT_ROW_NUMBER_FIELD, + TABULAR_EXPORT_OUTPUT_ROW_IDENTITY_FIELD, + } + }) + normalized_entries.append(normalized_entry) + + output_schema = list(expected_output_schema or []) + if not output_schema and normalized_entries: + output_schema = list(normalized_entries[0]) + + expected_fields = set(output_schema) + for row_index, normalized_entry in enumerate(normalized_entries, start=1): + actual_fields = set(normalized_entry) + if actual_fields != expected_fields: + missing_fields = sorted(expected_fields - actual_fields) + unexpected_fields = sorted(actual_fields - expected_fields) + raise ValueError( + f'Generated output schema mismatch at row {row_index}; ' + f'missing={missing_fields}; unexpected={unexpected_fields}' + ) + + ordered_entries = [ + {field_name: entry.get(field_name) for field_name in output_schema} + for entry in normalized_entries + ] + return ordered_entries, output_schema + + +def _build_generated_batch_summary(entries): + entries = [entry for entry in (entries or []) if isinstance(entry, dict)] + summary = { + 'row_count': len(entries), + 'source_row_start': None, + 'source_row_end': None, + 'fields': {}, + } + if not entries: + return summary + + summary['source_row_start'] = _safe_int(entries[0].get(TABULAR_EXPORT_OUTPUT_ROW_NUMBER_FIELD)) or None + summary['source_row_end'] = _safe_int(entries[-1].get(TABULAR_EXPORT_OUTPUT_ROW_NUMBER_FIELD)) or None + field_names = [ + field_name + for field_name in entries[0] + if field_name not in { + TABULAR_EXPORT_OUTPUT_ROW_NUMBER_FIELD, + TABULAR_EXPORT_OUTPUT_ROW_IDENTITY_FIELD, + } + ][:TABULAR_EXPORT_SUMMARY_MAX_FIELDS] + for field_name in field_names: + populated_count = 0 + value_counts = Counter() + for entry in entries: + field_value = entry.get(field_name) + if field_value in (None, '', [], {}): + continue + populated_count += 1 + if isinstance(field_value, (str, int, float, bool)): + normalized_value = str(field_value).strip() + if normalized_value and len(normalized_value) <= 100: + value_counts[normalized_value] += 1 + + summary['fields'][field_name] = { + 'populated_count': populated_count, + 'empty_count': len(entries) - populated_count, + 'top_values': [ + {'value': value, 'count': count} + for value, count in value_counts.most_common(TABULAR_EXPORT_SUMMARY_MAX_VALUES_PER_FIELD) + ], + } + return summary + + +def _build_compact_post_run_summary(run): + batch_count = _safe_int(run.get('batch_count')) + row_count = _safe_int(run.get('row_count')) + aggregate_fields = {} + summarized_batch_count = 0 + for batch_number in range(1, batch_count + 1): + summary_blob_path = _output_summary_blob_path( + run.get('user_id'), + run.get('conversation_id'), + run.get('id'), + batch_number, + ) + if not _blob_exists(summary_blob_path): + continue + batch_summary = _download_json_blob(summary_blob_path) + if not isinstance(batch_summary, dict): + continue + summarized_batch_count += 1 + for field_name, field_summary in list((batch_summary.get('fields') or {}).items())[ + :TABULAR_EXPORT_SUMMARY_MAX_FIELDS + ]: + aggregate_field = aggregate_fields.setdefault(field_name, { + 'populated_count': 0, + 'empty_count': 0, + 'value_counts': Counter(), + }) + aggregate_field['populated_count'] += _safe_int(field_summary.get('populated_count')) + aggregate_field['empty_count'] += _safe_int(field_summary.get('empty_count')) + for value_summary in field_summary.get('top_values') or []: + value = str(value_summary.get('value') or '').strip() + if value: + aggregate_field['value_counts'][value] += _safe_int(value_summary.get('count')) + if len(aggregate_field['value_counts']) > TABULAR_EXPORT_SUMMARY_AGGREGATE_MAX_VALUES * 2: + aggregate_field['value_counts'] = Counter(dict( + aggregate_field['value_counts'].most_common(TABULAR_EXPORT_SUMMARY_AGGREGATE_MAX_VALUES) + )) + + summary_parts = [ + f'Processed {row_count:,} ordered row(s) across {batch_count:,} checkpointed batch(es).' + ] + if aggregate_fields: + field_names = list(aggregate_fields)[:10] + summary_parts.append(f"Output fields: {', '.join(field_names)}.") + completeness_parts = [] + for field_name in field_names[:5]: + populated_count = aggregate_fields[field_name]['populated_count'] + completeness_percent = round((populated_count / row_count) * 100) if row_count else 0 + completeness_parts.append(f'{field_name} {completeness_percent}% populated') + if completeness_parts: + summary_parts.append(f"Completeness: {', '.join(completeness_parts)}.") + + common_value_parts = [] + for field_name in field_names[:5]: + top_values = aggregate_fields[field_name]['value_counts'].most_common(3) + if 1 < len(top_values) <= 3: + rendered_values = ', '.join(f'{value} ({count:,})' for value, count in top_values) + common_value_parts.append(f'{field_name}: {rendered_values}') + if common_value_parts: + summary_parts.append(f"Common values: {'; '.join(common_value_parts)}.") + + if summarized_batch_count != batch_count: + summary_parts.append( + f'Batch summaries available for {summarized_batch_count:,} of {batch_count:,} batch(es).' + ) + return ' '.join(summary_parts)[:2000] + + def _build_generated_output_csv(entries): ordered_columns = [] seen_columns = set() @@ -252,14 +544,18 @@ def _output_blob_path(user_id, conversation_id, run_id, batch_number): return f"{user_id}/{conversation_id}/generated/tabular_runs/{run_id}/output/batch_{batch_number:06d}.json" -def _upload_json_blob(blob_path, payload, metadata=None): +def _output_summary_blob_path(user_id, conversation_id, run_id, batch_number): + return f"{user_id}/{conversation_id}/generated/tabular_runs/{run_id}/summary/batch_{batch_number:06d}.json" + + +def _upload_json_blob(blob_path, payload, metadata=None, overwrite=True): blob_client = _get_blob_service_client().get_blob_client( container=storage_account_personal_chat_container_name, blob=blob_path, ) blob_client.upload_blob( json.dumps(payload, default=str, ensure_ascii=False).encode('utf-8'), - overwrite=True, + overwrite=overwrite, metadata={str(key): str(value) for key, value in (metadata or {}).items()}, ) @@ -283,6 +579,109 @@ def _blob_exists(blob_path): return bool(blob_client.exists()) +def _delete_blob_if_exists(blob_path): + blob_client = _get_blob_service_client().get_blob_client( + container=storage_account_personal_chat_container_name, + blob=blob_path, + ) + if blob_client.exists(): + blob_client.delete_blob() + + +def _authorize_tabular_export_run_execution(run): + user_id = str((run or {}).get('user_id') or '').strip() + conversation_id = str((run or {}).get('conversation_id') or '').strip() + if not user_id or not conversation_id: + raise PermissionError('Export run identity is incomplete') + + try: + conversation = cosmos_conversations_container.read_item( + item=conversation_id, + partition_key=conversation_id, + ) + except CosmosResourceNotFoundError as exc: + raise PermissionError('Export conversation no longer exists') from exc + if str(conversation.get('user_id') or '').strip() != user_id: + raise PermissionError('Export conversation ownership changed') + + source_authorization = ( + (run or {}).get('source_descriptor') + or (run or {}).get('source_authorization') + or {} + ) + if not source_authorization: + return conversation + + source = str(source_authorization.get('source') or '').strip().lower() + container_name = str(source_authorization.get('container') or '').strip() + blob_path = str(source_authorization.get('blob_path') or '').strip() + scope_id = str(source_authorization.get('scope_id') or '').strip() + if not source: + raise PermissionError('Export source authorization is incomplete') + + if source == 'chat': + expected_prefix = f'{user_id}/{conversation_id}/' + authorized = not container_name and not blob_path or ( + container_name == storage_account_personal_chat_container_name + and blob_path.startswith(expected_prefix) + ) + elif source == 'workspace': + expected_prefix = f'{user_id}/' + authorized = not container_name and not blob_path or ( + container_name == storage_account_user_documents_container_name + and blob_path.startswith(expected_prefix) + ) + elif source == 'group': + if not scope_id: + raise PermissionError('Export group scope is missing') + assert_group_role( + user_id, + scope_id, + allowed_roles=('Owner', 'Admin', 'DocumentManager', 'User'), + ) + authorized = not container_name and not blob_path or ( + container_name == storage_account_group_documents_container_name + and blob_path.startswith(f'{scope_id}/') + ) + elif source == 'public': + if not scope_id: + raise PermissionError('Export public workspace scope is missing') + visible_workspace_ids = set(get_user_visible_public_workspace_ids_from_settings(user_id) or []) + authorized = scope_id in visible_workspace_ids and ( + not container_name and not blob_path + or ( + container_name == storage_account_public_documents_container_name + and blob_path.startswith(f'{scope_id}/') + ) + ) + else: + raise PermissionError('Export source type is not supported') + + if not authorized: + raise PermissionError('Export source is no longer authorized') + return conversation + + +def _get_versioned_source_blob_client(source_descriptor): + container_name = str((source_descriptor or {}).get('container') or '').strip() + blob_path = str((source_descriptor or {}).get('blob_path') or '').strip() + expected_etag = str((source_descriptor or {}).get('blob_etag') or '').strip() + if not container_name or not blob_path or not expected_etag: + raise ValueError('Source query descriptor is incomplete') + + blob_client = _get_blob_service_client().get_blob_client( + container=container_name, + blob=blob_path, + ) + blob_properties = blob_client.get_blob_properties() + current_etag = str(getattr(blob_properties, 'etag', '') or '').strip() + if isinstance(blob_properties, dict): + current_etag = current_etag or str(blob_properties.get('etag') or '').strip() + if current_etag != expected_etag: + raise ValueError('Source CSV changed after the export was queued') + return blob_client + + def _clean_generated_json_code_fence(response_content): cleaned = str(response_content or '').strip() if not cleaned: @@ -336,17 +735,286 @@ def _dump_generated_output_json(value): return json.dumps(value, default=str, ensure_ascii=False, separators=(',', ':')) -def _build_batch_prompt(user_question, batch_rows, batch_index, total_batches, source_file_name, selected_sheet=''): +def _checkpoint_source_input_batch(run, batch_rows, source_scan_row_count): + _raise_if_tabular_export_canceled(run) + staged_row_count = _safe_int(run.get('source_staged_rows')) + batch_number = _safe_int(run.get('source_staged_batches')) + 1 + prepared_rows = _prepare_tabular_source_rows( + batch_rows, + start_row=staged_row_count, + token_namespace=run.get('id'), + ) + input_blob_path = _input_blob_path( + run.get('user_id'), + run.get('conversation_id'), + run.get('id'), + batch_number, + ) + _upload_json_blob( + input_blob_path, + prepared_rows, + metadata={ + 'run_id': run.get('id'), + 'conversation_id': run.get('conversation_id'), + 'generated_output_input': 'true', + 'batch_number': batch_number, + 'source_query_checkpoint': 'true', + }, + ) + _raise_if_tabular_export_canceled(run) + + now = _now_iso() + run.update({ + 'source_staged_rows': staged_row_count + len(prepared_rows), + 'source_staged_batches': batch_number, + 'source_scan_row_count': _safe_int(source_scan_row_count), + 'updated_at': now, + 'last_heartbeat_at': now, + 'last_message': ( + f'Staged source query batch {batch_number} ' + f'with {staged_row_count + len(prepared_rows)} row(s) ready' + ), + }) + return _replace_claimed_run(run) + + +def _stage_tabular_generated_output_source(run, settings): + source_descriptor = run.get('source_descriptor') or {} + if source_descriptor.get('kind') != 'query_tabular_data': + raise ValueError('Unsupported generated export source descriptor') + if not str(source_descriptor.get('blob_path') or '').lower().endswith('.csv'): + raise ValueError('Source-backed generated exports require a CSV source') + + expected_row_count = _safe_int(source_descriptor.get('expected_row_count')) + if expected_row_count <= 0: + raise ValueError('Source query descriptor has no expected rows') + source_chunk_rows = _settings_int( + settings, + 'tabular_generated_output_source_chunk_rows', + TABULAR_EXPORT_DEFAULT_SOURCE_CHUNK_ROWS, + minimum=100, + maximum=10000, + ) + max_batch_rows = _safe_int( + source_descriptor.get('batch_max_rows'), + default=TABULAR_EXPORT_DEFAULT_SOURCE_BATCH_ROWS, + minimum=1, + maximum=100, + ) + max_batch_chars = _safe_int( + source_descriptor.get('batch_max_chars'), + default=TABULAR_EXPORT_DEFAULT_SOURCE_BATCH_CHARS, + minimum=6000, + maximum=120000, + ) + source_blob_client = _get_versioned_source_blob_client(source_descriptor) + resume_source_row = _safe_int(run.get('source_scan_row_count')) + pending_rows = [] + pending_chars = 0 + last_source_row_number = resume_source_row + + with tempfile.SpooledTemporaryFile( + max_size=TABULAR_EXPORT_FINAL_SPOOL_MAX_MEMORY_BYTES, + mode='w+b', + ) as source_stream: + source_blob_client.download_blob( + etag=source_descriptor.get('blob_etag'), + match_condition=MatchConditions.IfNotModified, + ).readinto(source_stream) + source_stream.seek(0) + + tabular_plugin = TabularProcessingPlugin() + for source_row_number, source_row in iter_tabular_csv_query_rows( + csv_stream=source_stream, + query_expression=source_descriptor.get('query_expression'), + return_columns=source_descriptor.get('return_columns'), + source_chunk_rows=source_chunk_rows, + tabular_plugin=tabular_plugin, + start_source_row=resume_source_row, + ): + source_row_text = _dump_generated_output_json(source_row) + if pending_rows and ( + len(pending_rows) >= max_batch_rows + or pending_chars + len(source_row_text) > max_batch_chars + ): + run = _checkpoint_source_input_batch( + run, + pending_rows, + source_scan_row_count=source_row_number - 1, + ) + pending_rows = [] + pending_chars = 0 + + pending_rows.append(source_row) + pending_chars += len(source_row_text) + last_source_row_number = source_row_number + if len(pending_rows) >= max_batch_rows: + run = _checkpoint_source_input_batch( + run, + pending_rows, + source_scan_row_count=source_row_number, + ) + pending_rows = [] + pending_chars = 0 + + if pending_rows: + run = _checkpoint_source_input_batch( + run, + pending_rows, + source_scan_row_count=last_source_row_number, + ) + + staged_row_count = _safe_int(run.get('source_staged_rows')) + staged_batch_count = _safe_int(run.get('source_staged_batches')) + if staged_row_count != expected_row_count: + raise ValueError( + f'Source query returned {staged_row_count} row(s); expected {expected_row_count}' + ) + if staged_batch_count <= 0: + raise ValueError('Source query produced no input checkpoints') + + now = _now_iso() + run.update({ + 'source_staging_complete': True, + 'row_count': staged_row_count, + 'batch_count': staged_batch_count, + 'updated_at': now, + 'last_heartbeat_at': now, + 'last_message': ( + f'Source query staging complete: {staged_row_count} row(s) ' + f'across {staged_batch_count} batch(es)' + ), + }) + return _replace_claimed_run(run) + + +def _migrate_legacy_tabular_export_run(run): + if _safe_int(run.get('contract_version')) >= TABULAR_EXPORT_CONTRACT_VERSION: + return run + + batch_count = _safe_int(run.get('batch_count')) + expected_row_count = _safe_int(run.get('row_count')) + if batch_count <= 0 or expected_row_count <= 0: + raise ValueError('Legacy export run has invalid input counts') + + aggregate_input_batches = None + aggregate_input_blob_path = str(run.get('input_blob_path') or '').strip() + if aggregate_input_blob_path: + aggregate_input_batches = _download_json_blob(aggregate_input_blob_path) + if not isinstance(aggregate_input_batches, list): + raise ValueError('Legacy input batches blob was not a JSON array') + + migrated_row_count = 0 + for batch_number in range(1, batch_count + 1): + _raise_if_tabular_export_canceled(run) + if aggregate_input_batches is not None: + try: + batch_rows = aggregate_input_batches[batch_number - 1] + except IndexError as exc: + raise ValueError(f'Legacy input batch {batch_number}/{batch_count} is missing') from exc + else: + batch_rows = _download_json_blob(_input_blob_path( + run.get('user_id'), + run.get('conversation_id'), + run.get('id'), + batch_number, + )) + if not isinstance(batch_rows, list): + raise ValueError(f'Legacy input batch {batch_number}/{batch_count} was not a JSON array') + + prepared_rows = _prepare_tabular_source_rows( + batch_rows, + start_row=migrated_row_count, + token_namespace=run.get('id'), + ) + _upload_json_blob( + _input_blob_path( + run.get('user_id'), + run.get('conversation_id'), + run.get('id'), + batch_number, + ), + prepared_rows, + metadata={ + 'run_id': run.get('id'), + 'conversation_id': run.get('conversation_id'), + 'generated_output_input': 'true', + 'batch_number': batch_number, + 'contract_version': TABULAR_EXPORT_CONTRACT_VERSION, + }, + ) + _delete_blob_if_exists(_output_blob_path( + run.get('user_id'), + run.get('conversation_id'), + run.get('id'), + batch_number, + )) + _delete_blob_if_exists(_output_summary_blob_path( + run.get('user_id'), + run.get('conversation_id'), + run.get('id'), + batch_number, + )) + migrated_row_count += len(prepared_rows) + + if migrated_row_count != expected_row_count: + raise ValueError( + f'Legacy input migration found {migrated_row_count} row(s); expected {expected_row_count}' + ) + + now = _now_iso() + run.update({ + 'contract_version': TABULAR_EXPORT_CONTRACT_VERSION, + 'input_blob_path': None, + 'completed_batches': 0, + 'processed_rows': 0, + 'output_schema': None, + 'regenerate_legacy_output_checkpoints': False, + 'updated_at': now, + 'last_heartbeat_at': now, + 'last_message': 'Legacy export inputs migrated; output checkpoints will be regenerated', + }) + _raise_if_tabular_export_canceled(run) + return _replace_claimed_run(run) + + +def _build_batch_prompt( + user_question, + batch_rows, + batch_index, + total_batches, + source_file_name, + selected_sheet='', + output_schema=None, +): source_file_name = str(source_file_name or 'unknown file').strip() or 'unknown file' selected_sheet = str(selected_sheet or '').strip() batch_rows_json = _dump_generated_output_json(batch_rows) selected_sheet_line = f"Worksheet: {selected_sheet}\n" if selected_sheet else '' + model_output_schema = [ + field_name + for field_name in (output_schema or []) + if field_name not in { + TABULAR_EXPORT_OUTPUT_ROW_NUMBER_FIELD, + TABULAR_EXPORT_OUTPUT_ROW_IDENTITY_FIELD, + } + ] + output_schema_line = ( + f'Use exactly these output fields for every object, in this order: ' + f'{json.dumps(model_output_schema, ensure_ascii=False)}.\n' + if model_output_schema + else '' + ) return ( 'Transform the tabular input rows below into structured output for the user.\n\n' f'User instructions:\n{user_question}\n\n' 'Return ONLY a valid JSON array.\n' f'Return exactly {len(batch_rows)} JSON object(s), one per input row, in the same order.\n' + f'{output_schema_line}' + f'Copy {TABULAR_EXPORT_INPUT_ROW_TOKEN_FIELD} exactly from each input row into its matching output object. ' + f'The {TABULAR_EXPORT_INPUT_ROW_NUMBER_FIELD} and {TABULAR_EXPORT_INPUT_ROW_IDENTITY_FIELD} fields are internal; ' + 'do not include those two fields in generated objects.\n' 'Do not drop, merge, summarize, or cap rows.\n' 'Input rows may include normalized helper fields such as comment_id, body_text, source_file, attachment_present, attachment_names, and attachment_text. Use those normalized fields when they are present.\n' 'Input rows may include a referenced_documents array containing row-linked evidence from explicitly referenced non-tabular documents. Use that evidence as part of the source row context when it is relevant to the requested output.\n' @@ -380,6 +1048,7 @@ async def _generate_batch_entries( selected_sheet, retry_attempts, run_id, + expected_output_schema=None, ): batch_number = batch_index + 1 batch_prompt = _build_batch_prompt( @@ -389,11 +1058,13 @@ async def _generate_batch_entries( total_batches, source_file_name, selected_sheet=selected_sheet, + output_schema=expected_output_schema, ) parsed_entries = None raw_response_content = '' mismatch_count = 0 + last_validation_error = None for attempt_number in range(1, retry_attempts + 1): chat_history = SKChatHistory() chat_history.add_system_message( @@ -414,7 +1085,15 @@ async def _generate_batch_entries( parsed_entries = _parse_generated_json_entries(raw_response_content) if raw_response_content else None parsed_entry_count = len(parsed_entries) if parsed_entries is not None else 0 if parsed_entries is not None and parsed_entry_count == len(batch_rows): - return parsed_entries, mismatch_count + try: + normalized_entries, output_schema = _normalize_generated_batch_entries( + batch_rows, + parsed_entries, + expected_output_schema=expected_output_schema, + ) + return normalized_entries, mismatch_count, output_schema + except ValueError as exc: + last_validation_error = str(exc) mismatch_count += 1 log_event( @@ -426,15 +1105,19 @@ async def _generate_batch_entries( 'attempt_number': attempt_number, 'expected_row_count': len(batch_rows), 'parsed_row_count': parsed_entry_count, + 'validation_error': last_validation_error, 'response_char_count': len(raw_response_content), 'response_preview': _truncate_response_preview(raw_response_content), }, debug_only=True, ) + failure_detail = last_validation_error or ( + f'returned {len(parsed_entries) if parsed_entries is not None else 0} object(s) ' + f'for {len(batch_rows)} input row(s)' + ) raise ValueError( - f'Background structured export batch {batch_number}/{total_batches} returned ' - f'{len(parsed_entries) if parsed_entries is not None else 0} object(s) for {len(batch_rows)} input row(s).' + f'Background structured export batch {batch_number}/{total_batches} failed validation: {failure_detail}.' ) @@ -448,10 +1131,11 @@ async def _generate_batch_entries_for_window( selected_sheet, retry_attempts, run_id, + expected_output_schema, ): async with semaphore: batch_started_at = time.monotonic() - batch_entries, mismatch_count = await _generate_batch_entries( + batch_entries, mismatch_count, output_schema = await _generate_batch_entries( chat_service, user_question, batch_request['rows'], @@ -461,13 +1145,16 @@ async def _generate_batch_entries_for_window( selected_sheet, retry_attempts, run_id, + expected_output_schema=expected_output_schema, ) return { 'batch_number': batch_request['batch_number'], 'batch_entries': batch_entries, + 'batch_summary': _build_generated_batch_summary(batch_entries), 'batch_row_count': len(batch_entries), 'elapsed_seconds': time.monotonic() - batch_started_at, 'mismatch_count': mismatch_count, + 'output_schema': output_schema, } @@ -481,6 +1168,7 @@ async def _generate_batch_window_entries( retry_attempts, run_id, batch_concurrency, + expected_output_schema=None, ): semaphore = asyncio.Semaphore(max(1, batch_concurrency)) tasks = [ @@ -494,6 +1182,7 @@ async def _generate_batch_window_entries( selected_sheet, retry_attempts, run_id, + expected_output_schema, ) for batch_request in batch_requests ] @@ -628,23 +1317,32 @@ def _scheduler_candidate_sort_key(run): ) -def _query_scheduler_candidates_by_status(status, scan_limit): +def _query_scheduler_candidates_by_status(status, scan_limit, settings): per_status_limit = _safe_int(scan_limit, default=TABULAR_EXPORT_DEFAULT_SCAN_LIMIT, minimum=1, maximum=10) query = ( - f"SELECT TOP {per_status_limit} " + "SELECT " "c.id, c.user_id, c.status, c.created_at, c.updated_at, c.last_heartbeat_at, " "c.next_attempt_at, c.last_error, c.transient_failure_count " - "FROM c WHERE c.type = @type AND c.status = @status" + "FROM c WHERE c.type = @type AND c.status = @status " + "ORDER BY c.updated_at ASC" ) try: - return list(cosmos_tabular_export_runs_container.query_items( + query_results = cosmos_tabular_export_runs_container.query_items( query=query, parameters=[ {'name': '@type', 'value': TABULAR_EXPORT_RUN_TYPE}, {'name': '@status', 'value': status}, ], enable_cross_partition_query=True, - )) + ) + eligible_candidates = [] + for run in query_results: + if not _scheduler_candidate_reason(run, settings): + continue + eligible_candidates.append(run) + if len(eligible_candidates) >= per_status_limit: + break + return eligible_candidates except Exception as exc: log_event( '[Tabular Generated Output] Scheduler candidate query failed', @@ -679,6 +1377,14 @@ def _can_resume_run(run, settings=None): return False +def _can_cancel_run(run): + status = str((run or {}).get('status') or '').strip().lower() + return not run.get('publishing_started_at') and status not in { + TABULAR_EXPORT_STATUS_COMPLETED, + TABULAR_EXPORT_STATUS_CANCELED, + } + + def _build_checkpoint_summary(completed_batches, batch_count, processed_rows, row_count): checkpoint_parts = [] if batch_count: @@ -727,6 +1433,16 @@ def _build_run_status_detail(run, settings, retryable_failure, can_resume): 'retry_delay_seconds': None, } if status == TABULAR_EXPORT_STATUS_RUNNING: + if run.get('publishing_started_at'): + return { + 'status_label': 'Finalizing', + 'status_tone': 'info', + 'status_detail': 'Export validation passed and the final artifact is being published.', + 'is_stale': False, + 'waiting_for_retry': False, + 'retry_due': False, + 'retry_delay_seconds': None, + } return { 'status_label': 'Running', 'status_tone': 'info', @@ -828,6 +1544,8 @@ def _build_run_public_status(run, settings=None): 'storage_scope': 'chat', 'source_file_name': run.get('source_file_name'), 'selected_sheet': run.get('selected_sheet'), + 'summary': run.get('post_run_summary'), + 'suppress_assistant_table_export': True, } return { @@ -865,11 +1583,13 @@ def _build_run_public_status(run, settings=None): 'manual_resume_count': _safe_int(run.get('manual_resume_count')), 'next_attempt_at': run.get('next_attempt_at'), 'can_resume': can_resume, + 'can_cancel': _can_cancel_run(run), 'retryable_failure': retryable_failure, 'artifact_message_id': final_artifact.get('artifact_message_id'), 'file_name': final_artifact.get('file_name') or run.get('generated_file_name'), 'generated_artifact': generated_artifact, 'capability': 'tabular', + 'suppress_assistant_table_export': True, 'background_export': not ( str(run.get('status') or '').strip().lower() == TABULAR_EXPORT_STATUS_COMPLETED and generated_artifact @@ -884,6 +1604,7 @@ def build_background_tabular_generated_output_metadata(run): 'export_run_id': public_status.get('run_id'), 'background_export': True, 'capability': 'tabular', + 'suppress_assistant_table_export': True, 'summary': ( f"Queued structured {str(public_status.get('output_format') or 'json').upper()} export " f"for {public_status.get('row_count', 0)} row(s) across {public_status.get('batch_count', 0)} batch(es)." @@ -922,6 +1643,18 @@ def resume_tabular_generated_output_run(user_id, run_id): except CosmosResourceNotFoundError: return None + try: + _authorize_tabular_export_run_execution(run) + except (LookupError, PermissionError, ValueError): + return { + 'success': False, + 'resumed': False, + 'submitted': False, + 'authorization_failed': True, + 'message': 'Background export access is no longer authorized.', + 'run': _build_run_public_status(run, settings=settings), + } + status = str(run.get('status') or '').strip().lower() if status == TABULAR_EXPORT_STATUS_COMPLETED: return { @@ -939,6 +1672,14 @@ def resume_tabular_generated_output_run(user_id, run_id): 'message': 'Canceled background exports cannot be continued.', 'run': _build_run_public_status(run, settings=settings), } + if not _can_cancel_run(run): + return { + 'success': False, + 'resumed': False, + 'submitted': False, + 'message': 'Background export is already publishing and cannot be resumed.', + 'run': _build_run_public_status(run, settings=settings), + } if status == TABULAR_EXPORT_STATUS_RUNNING and not _is_stale_running_run(run, settings): return { 'success': True, @@ -970,10 +1711,21 @@ def resume_tabular_generated_output_run(user_id, run_id): 'manual_resume_count': _safe_int(run.get('manual_resume_count')) + 1, 'last_manual_resume_at': now, }) - run = _upsert_run(run) + try: + run = _replace_run(run) + except Exception as exc: + if getattr(exc, 'status_code', None) not in (409, 412): + raise + current_run = _read_run(normalized_user_id, normalized_run_id) + return { + 'success': False, + 'resumed': False, + 'submitted': False, + 'message': 'Background export state changed. Refresh its status before continuing.', + 'run': _build_run_public_status(current_run, settings=settings), + } submitted = submit_tabular_generated_output_run(normalized_run_id, normalized_user_id) run['submitted_to_executor'] = submitted - run = _upsert_run(run) log_event( '[Tabular Generated Output] Background export manually resumed', { @@ -998,6 +1750,86 @@ def resume_tabular_generated_output_run(user_id, run_id): } +def cancel_tabular_generated_output_run(user_id, run_id): + """Cancel a queued, running, retryable, or failed generated-output run.""" + normalized_user_id = str(user_id or '').strip() + normalized_run_id = str(run_id or '').strip() + if not normalized_user_id or not normalized_run_id: + return None + + settings = get_settings() + try: + run = _read_run(normalized_user_id, normalized_run_id) + except CosmosResourceNotFoundError: + return None + + status = str(run.get('status') or '').strip().lower() + if status == TABULAR_EXPORT_STATUS_COMPLETED: + return { + 'success': False, + 'canceled': False, + 'message': 'Completed background exports cannot be canceled.', + 'run': _build_run_public_status(run, settings=settings), + } + if status == TABULAR_EXPORT_STATUS_CANCELED: + return { + 'success': True, + 'canceled': True, + 'message': 'Background export is already canceled.', + 'run': _build_run_public_status(run, settings=settings), + } + if not _can_cancel_run(run): + return { + 'success': False, + 'canceled': False, + 'message': 'Background export is already publishing and can no longer be canceled.', + 'run': _build_run_public_status(run, settings=settings), + } + + now = _now_iso() + run.update({ + 'status': TABULAR_EXPORT_STATUS_CANCELED, + 'updated_at': now, + 'completed_at': now, + 'last_heartbeat_at': now, + 'lease_holder_id': None, + 'lease_expires_at': None, + 'next_attempt_at': None, + 'last_message': 'Background structured export canceled by the user', + 'last_error': None, + 'canceled_at': now, + }) + try: + run = _replace_run(run) + except Exception as exc: + if getattr(exc, 'status_code', None) not in (409, 412): + raise + current_run = _read_run(normalized_user_id, normalized_run_id) + return { + 'success': False, + 'canceled': False, + 'message': 'Background export state changed. Refresh its status before canceling.', + 'run': _build_run_public_status(current_run, settings=settings), + } + log_event( + '[Tabular Generated Output] Background export canceled', + { + 'run_id': normalized_run_id, + 'conversation_id': run.get('conversation_id'), + 'user_id': normalized_user_id, + 'completed_batches': run.get('completed_batches'), + 'processed_rows': run.get('processed_rows'), + }, + level=logging.INFO, + ) + return { + 'success': True, + 'canceled': True, + 'message': 'Background export canceled.', + 'run': _build_run_public_status(run, settings=settings), + } + + def _read_run(user_id, run_id): return cosmos_tabular_export_runs_container.read_item( item=run_id, @@ -1005,6 +1837,37 @@ def _read_run(user_id, run_id): ) +def _raise_if_tabular_export_canceled(run): + current_run = _read_run(run.get('user_id'), run.get('id')) + current_status = str(current_run.get('status') or '').strip().lower() + if current_status == TABULAR_EXPORT_STATUS_CANCELED: + run.clear() + run.update(current_run) + raise TabularExportCanceledError('Background structured export was canceled') + + claim_matches = ( + current_status == TABULAR_EXPORT_STATUS_RUNNING + and str(current_run.get('lease_holder_id') or '') == str(run.get('lease_holder_id') or '') + and _safe_int(current_run.get('lease_generation')) == _safe_int(run.get('lease_generation')) + ) + if not claim_matches: + raise TabularExportLeaseLostError('Background structured export worker lost its claim') + + run['_etag'] = current_run.get('_etag') + return current_run + + +def _replace_claimed_run(run): + try: + return _replace_run(run) + except Exception as exc: + if getattr(exc, 'status_code', None) in (409, 412): + raise TabularExportLeaseLostError( + 'Background structured export worker lost its claim' + ) from exc + raise + + def _replace_run(run): return cosmos_tabular_export_runs_container.replace_item( item=run.get('id'), @@ -1014,10 +1877,6 @@ def _replace_run(run): ) -def _upsert_run(run): - return cosmos_tabular_export_runs_container.upsert_item(run) - - def _lease_holder_id(): return f"{socket.gethostname()}:{os.getpid()}:{uuid.uuid4().hex}" @@ -1080,6 +1939,7 @@ def _try_claim_run(user_id, run_id, settings): 'completed_at': None, 'last_heartbeat_at': now.isoformat(), 'lease_holder_id': _lease_holder_id(), + 'lease_generation': _safe_int(run.get('lease_generation')) + 1, 'lease_expires_at': (now + timedelta(seconds=lease_seconds)).isoformat(), 'next_attempt_at': None, 'last_message': 'Background structured export is running', @@ -1107,7 +1967,10 @@ def _mark_run_failed(run, error_message): 'last_error': str(error_message or 'Unknown error')[:1000], 'last_message': 'Background structured export failed', }) - _upsert_run(run) + try: + run = _replace_claimed_run(run) + except TabularExportLeaseLostError: + return _read_run(run.get('user_id'), run.get('id')) log_event( '[Tabular Generated Output] Background export run failed', { @@ -1156,7 +2019,10 @@ def _mark_run_retryable(run, error_message, settings): 'transient_failure_count': transient_failure_count, 'next_attempt_at': next_attempt_at, }) - _upsert_run(run) + try: + run = _replace_claimed_run(run) + except TabularExportLeaseLostError: + return _read_run(run.get('user_id'), run.get('id')) log_event( '[Tabular Generated Output] Background export run requeued after transient failure', { @@ -1231,7 +2097,7 @@ def _update_run_progress(run, completed_batches, processed_rows, batch_rows, bat }) run['recent_batches'] = recent_batches - return _upsert_run(run) + return _replace_claimed_run(run) def _log_progress_if_due(run, last_logged_at): @@ -1263,44 +2129,122 @@ def _log_progress_if_due(run, last_logged_at): return now_monotonic -def _assemble_output_entries(run): - output_entries = [] +def _write_ordered_output_stream(run, output_stream): user_id = run.get('user_id') conversation_id = run.get('conversation_id') run_id = run.get('id') batch_count = _safe_int(run.get('batch_count')) + expected_row_count = _safe_int(run.get('row_count')) + output_format = str(run.get('output_format') or 'json').strip().lower() or 'json' + output_schema = list(run.get('output_schema') or []) + if not output_schema: + raise ValueError('Generated output schema is missing') + if TABULAR_EXPORT_OUTPUT_ROW_NUMBER_FIELD not in output_schema: + raise ValueError('Generated output schema is missing source row order') + + csv_writer = None + if output_format == 'csv': + csv_writer = csv.DictWriter(output_stream, fieldnames=output_schema, lineterminator='\n') + csv_writer.writeheader() + else: + output_stream.write('[\n') + + written_row_count = 0 + expected_source_row_number = 1 for batch_number in range(1, batch_count + 1): batch_blob_path = _output_blob_path(user_id, conversation_id, run_id, batch_number) batch_entries = _download_json_blob(batch_blob_path) - if isinstance(batch_entries, list): - output_entries.extend(batch_entries) - return output_entries + if not isinstance(batch_entries, list): + raise ValueError(f'Output checkpoint {batch_number}/{batch_count} was not a JSON array') + + for batch_row_index, entry in enumerate(batch_entries, start=1): + if not isinstance(entry, dict): + raise ValueError( + f'Output checkpoint {batch_number}/{batch_count} row {batch_row_index} was not an object' + ) + if set(entry) != set(output_schema): + raise ValueError( + f'Output checkpoint {batch_number}/{batch_count} row {batch_row_index} has schema drift' + ) + + source_row_number = _safe_int(entry.get(TABULAR_EXPORT_OUTPUT_ROW_NUMBER_FIELD)) + if source_row_number != expected_source_row_number: + raise ValueError( + f'Source row order gap or overlap: expected {expected_source_row_number}, ' + f'found {source_row_number}' + ) + ordered_entry = { + field_name: entry.get(field_name) + for field_name in output_schema + } + if csv_writer: + csv_writer.writerow({ + field_name: _serialize_generated_output_value(field_value) + for field_name, field_value in ordered_entry.items() + }) + else: + if written_row_count: + output_stream.write(',\n') + output_stream.write(json.dumps(ordered_entry, default=str, ensure_ascii=False)) + + written_row_count += 1 + expected_source_row_number += 1 + + if output_format != 'csv': + output_stream.write('\n]\n') + if written_row_count != expected_row_count: + raise ValueError( + f'Final output row count {written_row_count} does not match expected count {expected_row_count}' + ) + return written_row_count def _complete_run(run): - output_entries = _assemble_output_entries(run) output_format = str(run.get('output_format') or 'json').strip().lower() or 'json' - if output_format == 'csv': - serialized_output = _build_generated_output_csv(output_entries) - else: - serialized_output = json.dumps(output_entries, indent=2, default=str, ensure_ascii=False) - generated_file_name = run.get('generated_file_name') or _build_generated_file_name( run.get('source_file_name'), output_format, ) - upload_result = upload_generated_analysis_artifact_for_user( - current_user_id=run.get('user_id'), - conversation_id=run.get('conversation_id'), - file_name=generated_file_name, - file_content=serialized_output, - capability='tabular', - output_format=output_format, - summary=( - f"Saved {len(output_entries)} row(s) to {generated_file_name} " - 'from a durable background tabular export.' - ), - ) + with tempfile.SpooledTemporaryFile( + max_size=TABULAR_EXPORT_FINAL_SPOOL_MAX_MEMORY_BYTES, + mode='w+b', + ) as binary_output_stream: + text_output_stream = io.TextIOWrapper( + binary_output_stream, + encoding='utf-8', + newline='', + write_through=True, + ) + try: + output_entry_count = _write_ordered_output_stream(run, text_output_stream) + post_run_summary = _build_compact_post_run_summary(run) + _authorize_tabular_export_run_execution(run) + _raise_if_tabular_export_canceled(run) + if not run.get('publishing_started_at'): + run.update({ + 'publishing_started_at': _now_iso(), + 'last_message': 'Final validation passed; publishing the generated artifact', + }) + run = _replace_claimed_run(run) + _authorize_tabular_export_run_execution(run) + text_output_stream.flush() + output_size = binary_output_stream.tell() + binary_output_stream.seek(0) + upload_result = upload_generated_analysis_artifact_stream_for_user( + current_user_id=run.get('user_id'), + conversation_id=run.get('conversation_id'), + file_name=generated_file_name, + file_stream=binary_output_stream, + file_size=output_size, + capability='tabular', + output_format=output_format, + summary=post_run_summary, + artifact_idempotency_key=f"tabular-generated-output:{run.get('id')}", + ) + _raise_if_tabular_export_canceled(run) + finally: + text_output_stream.detach() + uploaded_message = upload_result.get('message') or {} now = _now_iso() run.update({ @@ -1308,9 +2252,10 @@ def _complete_run(run): 'updated_at': now, 'completed_at': now, 'last_heartbeat_at': now, - 'processed_rows': len(output_entries), + 'processed_rows': output_entry_count, 'completed_batches': _safe_int(run.get('batch_count')), 'last_message': 'Background structured export completed', + 'post_run_summary': post_run_summary, 'generated_file_name': uploaded_message.get('file_name') or generated_file_name, 'final_artifact': { 'artifact_message_id': uploaded_message.get('id'), @@ -1322,7 +2267,7 @@ def _complete_run(run): }, 'estimated_remaining_seconds': 0, }) - _upsert_run(run) + run = _replace_claimed_run(run) log_event( '[Tabular Generated Output] Background export completed', { @@ -1331,7 +2276,7 @@ def _complete_run(run): 'user_id': run.get('user_id'), 'source_file_name': run.get('source_file_name'), 'output_format': output_format, - 'row_count': len(output_entries), + 'row_count': output_entry_count, 'batch_count': run.get('batch_count'), 'artifact_message_id': uploaded_message.get('id'), 'generated_file_name': uploaded_message.get('file_name') or generated_file_name, @@ -1372,8 +2317,37 @@ def _build_batch_window(run, input_batches, user_id, run_id, window_start, windo batch_number, ) - if _blob_exists(output_blob_path): + if _blob_exists(output_blob_path) and not run.get('regenerate_legacy_output_checkpoints'): batch_entries = _download_json_blob(output_blob_path) + expected_output_schema = set(run.get('output_schema') or []) + if not isinstance(batch_entries, list) or not batch_entries: + raise ValueError(f'Output checkpoint {batch_number}/{batch_count} is empty or malformed') + for checkpoint_row_index, checkpoint_entry in enumerate(batch_entries, start=1): + if not isinstance(checkpoint_entry, dict): + raise ValueError( + f'Output checkpoint {batch_number}/{batch_count} row {checkpoint_row_index} is not an object' + ) + if set(checkpoint_entry) != expected_output_schema: + raise ValueError( + f'Output checkpoint {batch_number}/{batch_count} row {checkpoint_row_index} has schema drift' + ) + summary_blob_path = _output_summary_blob_path( + user_id, + run.get('conversation_id'), + run_id, + batch_number, + ) + if isinstance(batch_entries, list) and not _blob_exists(summary_blob_path): + _upload_json_blob( + summary_blob_path, + _build_generated_batch_summary(batch_entries), + metadata={ + 'run_id': run_id, + 'conversation_id': run.get('conversation_id'), + 'batch_number': batch_number, + 'generated_output_summary': 'true', + }, + ) batch_results[batch_number] = { 'batch_number': batch_number, 'batch_row_count': len(batch_entries) if isinstance(batch_entries, list) else 0, @@ -1404,8 +2378,29 @@ def _build_batch_window(run, input_batches, user_id, run_id, window_start, windo def _checkpoint_generated_batch_results(run, generated_results): + _raise_if_tabular_export_canceled(run) batch_results = {} - for generated_result in generated_results: + expected_output_schema = list(run.get('output_schema') or []) + ordered_results = sorted(generated_results, key=lambda result: result['batch_number']) + for generated_result in ordered_results: + generated_output_schema = list(generated_result.get('output_schema') or []) + if not expected_output_schema: + expected_output_schema = generated_output_schema + if generated_output_schema != expected_output_schema: + raise ValueError( + f"Generated output schema drifted in batch {generated_result['batch_number']}" + ) + + if not expected_output_schema: + raise ValueError('Generated output schema could not be established') + if list(run.get('output_schema') or []) != expected_output_schema: + run['output_schema'] = expected_output_schema + persisted_run = _replace_claimed_run(run) + run.clear() + run.update(persisted_run) + + for generated_result in ordered_results: + _raise_if_tabular_export_canceled(run) batch_number = generated_result['batch_number'] output_blob_path = _output_blob_path( run.get('user_id'), @@ -1413,14 +2408,49 @@ def _checkpoint_generated_batch_results(run, generated_results): run.get('id'), batch_number, ) + try: + _upload_json_blob( + output_blob_path, + generated_result['batch_entries'], + metadata={ + 'run_id': run.get('id'), + 'conversation_id': run.get('conversation_id'), + 'batch_number': batch_number, + 'generated_output': 'true', + 'lease_generation': run.get('lease_generation'), + }, + overwrite=bool(run.get('regenerate_legacy_output_checkpoints')), + ) + except ResourceExistsError: + checkpoint_entries = _download_json_blob(output_blob_path) + if ( + not isinstance(checkpoint_entries, list) + or len(checkpoint_entries) != generated_result['batch_row_count'] + or any( + not isinstance(entry, dict) or set(entry) != set(expected_output_schema) + for entry in checkpoint_entries + ) + ): + raise ValueError( + f'Concurrent output checkpoint {batch_number} failed schema or row-count validation' + ) + generated_result['batch_entries'] = checkpoint_entries + generated_result['batch_summary'] = _build_generated_batch_summary(checkpoint_entries) _upload_json_blob( - output_blob_path, - generated_result['batch_entries'], + _output_summary_blob_path( + run.get('user_id'), + run.get('conversation_id'), + run.get('id'), + batch_number, + ), + generated_result.get('batch_summary') or _build_generated_batch_summary( + generated_result['batch_entries'] + ), metadata={ 'run_id': run.get('id'), 'conversation_id': run.get('conversation_id'), 'batch_number': batch_number, - 'generated_output': 'true', + 'generated_output_summary': 'true', }, ) batch_results[batch_number] = { @@ -1467,6 +2497,11 @@ def process_tabular_generated_output_run(run_id, user_id): return None try: + _authorize_tabular_export_run_execution(run) + run = _migrate_legacy_tabular_export_run(run) + if run.get('source_descriptor') and not run.get('source_staging_complete'): + run = _stage_tabular_generated_output_source(run, settings) + retry_attempts = _settings_int( settings, 'tabular_generated_output_batch_retry_attempts', @@ -1514,8 +2549,11 @@ def process_tabular_generated_output_run(run_id, user_id): ) while completed_batches < batch_count: + _raise_if_tabular_export_canceled(run) window_start = completed_batches + 1 window_end = min(batch_count, window_start + batch_concurrency - 1) + if not run.get('output_schema'): + window_end = window_start batch_results, batch_requests = _build_batch_window( run, input_batches, @@ -1553,8 +2591,10 @@ def process_tabular_generated_output_run(run_id, user_id): retry_attempts, normalized_run_id, batch_concurrency, + expected_output_schema=run.get('output_schema'), ) ) + _raise_if_tabular_export_canceled(run) batch_results.update(_checkpoint_generated_batch_results(run, generated_results)) previous_completed_batches = completed_batches @@ -1572,7 +2612,12 @@ def process_tabular_generated_output_run(run_id, user_id): if completed_batches == previous_completed_batches: raise RuntimeError(f'No progress was made for batch window {window_start}-{window_end}') + _raise_if_tabular_export_canceled(run) return _complete_run(run) + except TabularExportCanceledError: + return _read_run(normalized_user_id, normalized_run_id) + except TabularExportLeaseLostError: + return _read_run(normalized_user_id, normalized_run_id) except Exception as exc: if _is_retryable_export_error(exc): return _mark_run_retryable(run, exc, settings) @@ -1609,6 +2654,7 @@ def queue_tabular_generated_output_run( gpt_model, settings=None, model_context=None, + source_descriptor=None, ): """Stage batch input blobs, create a run record, and submit background processing.""" normalized_user_id = str(user_id or '').strip() @@ -1622,34 +2668,80 @@ def queue_tabular_generated_output_run( selected_sheet = str(source_candidate.get('selected_sheet') or '').strip() normalized_output_format = str(output_format or 'json').strip().lower() or 'json' generated_file_name = _build_generated_file_name(source_file_name, normalized_output_format) - row_batches = list(row_batches or []) + settings = settings or {} + source_descriptor = dict(source_descriptor or {}) + source_authorization = dict(source_candidate.get('source_authorization') or {}) staged_row_count = 0 staged_char_count = 0 - normalized_row_batches = [] - - for index, batch_rows in enumerate(row_batches, start=1): - if not isinstance(batch_rows, list): - batch_rows = list(batch_rows or []) - normalized_row_batches.append(batch_rows) - staged_row_count += len(batch_rows) - staged_char_count += len(json.dumps(batch_rows, default=str, ensure_ascii=False)) + staged_batch_count = 0 + + if source_descriptor: + staged_row_count = _safe_int(source_descriptor.get('expected_row_count')) + if staged_row_count <= 0: + raise ValueError('Source query descriptor must include the expected row count') + source_descriptor['batch_max_rows'] = _safe_int( + source_descriptor.get('batch_max_rows'), + default=_settings_int( + settings, + 'tabular_generated_output_max_batch_rows', + TABULAR_EXPORT_DEFAULT_SOURCE_BATCH_ROWS, + minimum=1, + maximum=100, + ), + minimum=1, + maximum=100, + ) + source_descriptor['batch_max_chars'] = _safe_int( + source_descriptor.get('batch_max_chars'), + default=_settings_int( + settings, + 'tabular_generated_output_max_batch_chars', + TABULAR_EXPORT_DEFAULT_SOURCE_BATCH_CHARS, + minimum=6000, + maximum=120000, + ), + minimum=6000, + maximum=120000, + ) + staged_batch_count = max( + 1, + math.ceil(staged_row_count / source_descriptor['batch_max_rows']), + ) + source_authorization = { + field_name: source_descriptor.get(field_name) + for field_name in ('source', 'scope_id', 'container', 'blob_path') + } + else: + for index, batch_rows in enumerate(row_batches or [], start=1): + if not isinstance(batch_rows, list): + batch_rows = list(batch_rows or []) + prepared_batch_rows = _prepare_tabular_source_rows( + batch_rows, + start_row=staged_row_count, + token_namespace=run_id, + ) + _upload_json_blob( + _input_blob_path(normalized_user_id, normalized_conversation_id, run_id, index), + prepared_batch_rows, + metadata={ + 'run_id': run_id, + 'conversation_id': normalized_conversation_id, + 'generated_output_input': 'true', + 'batch_number': index, + }, + ) + staged_row_count += len(prepared_batch_rows) + staged_char_count += len(json.dumps(prepared_batch_rows, default=str, ensure_ascii=False)) + staged_batch_count = index - input_blob_path = _input_batches_blob_path(normalized_user_id, normalized_conversation_id, run_id) - _upload_json_blob( - input_blob_path, - normalized_row_batches, - metadata={ - 'run_id': run_id, - 'conversation_id': normalized_conversation_id, - 'generated_output_input': 'true', - 'batch_count': len(normalized_row_batches), - }, - ) + if not staged_batch_count or not staged_row_count: + raise ValueError('At least one source row is required for a background tabular export') now = _now_iso() run = { 'id': run_id, 'type': TABULAR_EXPORT_RUN_TYPE, + 'contract_version': TABULAR_EXPORT_CONTRACT_VERSION, 'user_id': normalized_user_id, 'conversation_id': normalized_conversation_id, 'status': TABULAR_EXPORT_STATUS_QUEUED, @@ -1666,11 +2758,18 @@ def queue_tabular_generated_output_run( 'model_context': model_context if isinstance(model_context, dict) else {}, 'generated_file_name': generated_file_name, 'row_count': staged_row_count, - 'batch_count': len(row_batches), + 'batch_count': staged_batch_count, 'completed_batches': 0, 'processed_rows': 0, + 'output_schema': None, + 'source_descriptor': source_descriptor or None, + 'source_authorization': source_authorization or None, + 'source_staging_complete': not bool(source_descriptor), + 'source_staged_rows': 0 if source_descriptor else staged_row_count, + 'source_staged_batches': 0 if source_descriptor else staged_batch_count, + 'source_scan_row_count': 0, 'input_blob_container': storage_account_personal_chat_container_name, - 'input_blob_path': input_blob_path, + 'input_blob_path': None, 'input_blob_prefix': f'{normalized_user_id}/{normalized_conversation_id}/generated/tabular_runs/{run_id}/input/', 'output_blob_container': storage_account_personal_chat_container_name, 'output_blob_prefix': f'{normalized_user_id}/{normalized_conversation_id}/generated/tabular_runs/{run_id}/output/', @@ -1697,8 +2796,9 @@ def queue_tabular_generated_output_run( 'selected_sheet': selected_sheet, 'output_format': normalized_output_format, 'row_count': staged_row_count, - 'batch_count': len(row_batches), + 'batch_count': staged_batch_count, 'staged_input_char_count': staged_char_count, + 'source_backed': bool(source_descriptor), 'submitted_to_executor': submitted, }, level=logging.INFO, @@ -1725,7 +2825,7 @@ def check_due_tabular_generated_output_runs_once(limit=None): scanned_candidates = [] status_counts = {} for status in TABULAR_EXPORT_SCHEDULER_STATUSES: - status_candidates = _query_scheduler_candidates_by_status(status, scan_limit) + status_candidates = _query_scheduler_candidates_by_status(status, scan_limit, settings) status_counts[status] = len(status_candidates) scanned_candidates.extend(status_candidates) @@ -1754,15 +2854,19 @@ def check_due_tabular_generated_output_runs_once(limit=None): for candidate in candidates: run = candidate.get('run') or {} status = str(run.get('status') or '').strip().lower() - processed_run = process_tabular_generated_output_run(run.get('id'), run.get('user_id')) - if processed_run: - processed.append(processed_run.get('id')) + submitted = submit_tabular_generated_output_run(run.get('id'), run.get('user_id')) + if submitted: + processed.append(run.get('id')) else: - skipped.append({ - 'run_id': run.get('id'), - 'status': status, - 'reason': f"{candidate.get('reason')}; claim or processing did not start", - }) + processed_run = process_tabular_generated_output_run(run.get('id'), run.get('user_id')) + if processed_run: + processed.append(processed_run.get('id')) + else: + skipped.append({ + 'run_id': run.get('id'), + 'status': status, + 'reason': f"{candidate.get('reason')}; claim or processing did not start", + }) if scanned_candidates or candidates: log_event( diff --git a/application/single_app/route_backend_chats.py b/application/single_app/route_backend_chats.py index e73a05c03..495bd9475 100644 --- a/application/single_app/route_backend_chats.py +++ b/application/single_app/route_backend_chats.py @@ -12,7 +12,11 @@ from semantic_kernel_plugins.plugin_invocation_thoughts import ( register_plugin_invocation_thought_callback, ) -from semantic_kernel_plugins.plugin_invocation_logger import get_plugin_logger, sanitize_plugin_invocation_value +from semantic_kernel_plugins.plugin_invocation_logger import ( + PluginInvocationResult, + get_plugin_logger, + sanitize_plugin_invocation_value, +) from semantic_kernel_plugins.chart_plugin import ChartPlugin from foundry_agent_runtime import FoundryAgentInvocationError, FoundryAgentUserAuthenticationRequired, execute_foundry_agent, resolve_authority from model_endpoint_clients import ( @@ -149,6 +153,7 @@ normalize_document_action_config, ) from functions_thoughts import ThoughtTracker +from functions_tabular_csv_query import validate_tabular_csv_query_expression from functions_workflow_runner import _execute_document_action_workflow from functions_simplechat_operations import ( derive_conversation_title_from_message, @@ -156,7 +161,10 @@ upload_generated_analysis_artifact_for_current_user, ) from functions_tabular_generated_exports import ( + _normalize_generated_batch_entries, + _prepare_tabular_source_rows, build_background_tabular_generated_output_metadata, + cancel_tabular_generated_output_run, get_tabular_generated_output_run_status, queue_tabular_generated_output_run, resume_tabular_generated_output_run, @@ -1223,7 +1231,13 @@ def _normalize_generated_analysis_artifact_metadata(raw_artifact, default_capabi artifact_message_id = str(raw_artifact.get('artifact_message_id') or '').strip() document_id = str(raw_artifact.get('document_id') or '').strip() export_run_id = str(raw_artifact.get('export_run_id') or raw_artifact.get('run_id') or '').strip() - if not artifact_message_id and not document_id and not export_run_id: + terminal_status = str(raw_artifact.get('status') or '').strip().lower() + suppress_assistant_table_export = bool(raw_artifact.get('suppress_assistant_table_export')) + is_terminal_export_status = ( + terminal_status in {'failed', 'canceled'} + and suppress_assistant_table_export + ) + if not artifact_message_id and not document_id and not export_run_id and not is_terminal_export_status: return None normalized_artifact = dict(raw_artifact) @@ -1238,6 +1252,9 @@ def _normalize_generated_analysis_artifact_metadata(raw_artifact, default_capabi if export_run_id: normalized_artifact['export_run_id'] = export_run_id normalized_artifact['background_export'] = bool(raw_artifact.get('background_export', True)) + elif is_terminal_export_status: + normalized_artifact['background_export'] = True + normalized_artifact['suppress_assistant_table_export'] = True normalized_output_format = str(raw_artifact.get('output_format') or '').strip().lower() if normalized_output_format: @@ -1357,6 +1374,10 @@ def _has_generated_tabular_csv_output(generated_outputs): continue capability = str(generated_output.get('capability') or '').strip().lower() + if generated_output.get('suppress_assistant_table_export') and ( + not capability or capability == 'tabular' + ): + return True output_format = str(generated_output.get('output_format') or '').strip().lower() file_name = str(generated_output.get('file_name') or '').strip().lower() if output_format == 'csv' or file_name.endswith('.csv'): @@ -3683,10 +3704,15 @@ def augment_tabular_invocations_with_related_document_evidence(invocations, user updated_result_payload['data'] = updated_rows updated_result_payload['referenced_document_row_count'] = augmented_rows_for_invocation updated_result_payload['referenced_document_match_count'] = augmented_documents_for_invocation - if isinstance(getattr(invocation, 'result', None), dict): + existing_result = getattr(invocation, 'result', None) + internal_metadata = getattr(existing_result, 'internal_metadata', {}) or {} + if isinstance(existing_result, dict): invocation.result = updated_result_payload else: - invocation.result = json.dumps(updated_result_payload, indent=2, default=str, ensure_ascii=False) + invocation.result = PluginInvocationResult( + json.dumps(updated_result_payload, indent=2, default=str, ensure_ascii=False), + internal_metadata=internal_metadata, + ) return { 'augmented_row_count': augmented_row_count, @@ -4319,6 +4345,9 @@ def _build_tabular_generated_output_candidate_diagnostic(invocation): 'full_result_available': full_result_available, 'function_rank': function_rank, 'max_rows': result_payload.get('max_rows') if isinstance(result_payload, dict) else None, + 'start_row': _safe_int(result_payload.get('start_row')) if isinstance(result_payload, dict) else 0, + 'has_more': bool(result_payload.get('has_more')) if isinstance(result_payload, dict) else False, + 'next_start_row': result_payload.get('next_start_row') if isinstance(result_payload, dict) else None, 'filter_applied': result_payload.get('filter_applied') if isinstance(result_payload, dict) else None, 'normalized_match': result_payload.get('normalized_match') if isinstance(result_payload, dict) else None, 'skip_reason': skip_reason, @@ -4333,50 +4362,327 @@ def _build_tabular_generated_output_candidate_diagnostics(invocations): ] +def _build_tabular_generated_output_source_signature(invocation, result_payload): + invocation_parameters = getattr(invocation, 'parameters', {}) or {} + invocation_internal_metadata = getattr( + getattr(invocation, 'result', None), + 'internal_metadata', + {}, + ) or {} + source_descriptor = invocation_internal_metadata.get('tabular_generated_export_source') or {} + source_authorization = invocation_internal_metadata.get('tabular_source_authorization') or {} + source_identity = { + 'source': source_descriptor.get('source') or source_authorization.get('source'), + 'scope_id': source_descriptor.get('scope_id') or source_authorization.get('scope_id'), + 'container': source_descriptor.get('container') or source_authorization.get('container'), + 'blob_path': source_descriptor.get('blob_path') or source_authorization.get('blob_path'), + 'blob_etag': source_descriptor.get('blob_etag') or source_authorization.get('blob_etag'), + } + signature_parameters = { + str(parameter_name): parameter_value + for parameter_name, parameter_value in invocation_parameters.items() + if str(parameter_name) not in { + 'user_id', + 'conversation_id', + 'start_row', + 'max_rows', + } + } + signature_payload = { + 'plugin_name': str(getattr(invocation, 'plugin_name', '') or '').strip(), + 'function_name': str(getattr(invocation, 'function_name', '') or '').strip(), + 'filename': result_payload.get('filename'), + 'selected_sheet': result_payload.get('selected_sheet'), + 'parameters': signature_parameters, + 'source_identity': source_identity, + } + return json.dumps(signature_payload, sort_keys=True, default=str, separators=(',', ':')) + + +def _coalesce_tabular_generated_output_pages(pages, total_matches): + normalized_total_matches = _safe_int(total_matches) + ordered_pages = sorted(pages or [], key=lambda page: page.get('start_row', 0)) + merged_rows = [] + next_expected_row = 0 + + for page in ordered_pages: + start_row = _safe_int(page.get('start_row')) + page_rows = page.get('rows') if isinstance(page.get('rows'), list) else [] + returned_rows = _safe_int(page.get('returned_rows')) + if returned_rows != len(page_rows): + return { + 'rows': merged_rows, + 'row_count': len(merged_rows), + 'full_result_available': False, + 'validation_error': ( + f'Page at row {start_row} declared {returned_rows} row(s) but contained {len(page_rows)}' + ), + } + if start_row < next_expected_row: + return { + 'rows': merged_rows, + 'row_count': len(merged_rows), + 'full_result_available': False, + 'validation_error': ( + f'Page overlap at row {start_row}; next expected row was {next_expected_row}' + ), + } + if start_row > next_expected_row: + return { + 'rows': merged_rows, + 'row_count': len(merged_rows), + 'full_result_available': False, + 'validation_error': ( + f'Page gap from row {next_expected_row} through {start_row - 1}' + ), + } + + merged_rows.extend(page_rows) + next_expected_row += len(page_rows) + + full_result_available = normalized_total_matches > 0 and next_expected_row == normalized_total_matches + validation_error = None + if next_expected_row < normalized_total_matches: + validation_error = ( + f'Page gap from row {next_expected_row} through {normalized_total_matches - 1}' + ) + elif normalized_total_matches and next_expected_row > normalized_total_matches: + validation_error = ( + f'Page coverage returned {next_expected_row} row(s) for {normalized_total_matches} total match(es)' + ) + + return { + 'rows': merged_rows, + 'row_count': len(merged_rows), + 'full_result_available': full_result_available, + 'validation_error': validation_error, + } + + def _build_tabular_generated_output_source_candidate(invocations): - best_candidate = None - best_score = None + candidate_groups = {} + logical_source_identities = {} for invocation in invocations or []: diagnostic = _build_tabular_generated_output_candidate_diagnostic(invocation) if diagnostic.get('skip_reason'): continue + result_payload = get_tabular_invocation_result_payload(invocation) + invocation_internal_metadata = getattr( + getattr(invocation, 'result', None), + 'internal_metadata', + {}, + ) or {} + source_signature = _build_tabular_generated_output_source_signature(invocation, result_payload) + logical_signature = json.dumps({ + 'plugin_name': str(getattr(invocation, 'plugin_name', '') or '').strip(), + 'function_name': str(getattr(invocation, 'function_name', '') or '').strip(), + 'filename': result_payload.get('filename'), + 'selected_sheet': result_payload.get('selected_sheet'), + 'parameters': { + str(parameter_name): parameter_value + for parameter_name, parameter_value in (getattr(invocation, 'parameters', {}) or {}).items() + if str(parameter_name) not in { + 'user_id', + 'conversation_id', + 'start_row', + 'max_rows', + } + }, + }, sort_keys=True, default=str, separators=(',', ':')) + logical_source_identities.setdefault(logical_signature, set()).add(source_signature) + candidate_group = candidate_groups.setdefault(source_signature, { + 'function_name': diagnostic.get('function_name'), + 'filename': result_payload.get('filename'), + 'selected_sheet': result_payload.get('selected_sheet'), + 'source_parameters': dict(getattr(invocation, 'parameters', {}) or {}), + 'source_descriptor': invocation_internal_metadata.get('tabular_generated_export_source'), + 'source_authorization': invocation_internal_metadata.get('tabular_source_authorization'), + 'function_rank': diagnostic.get('function_rank') or 0, + 'total_matches': diagnostic.get('total_matches') or 0, + 'pages': [], + 'diagnostics': [], + 'logical_signature': logical_signature, + }) + candidate_group['diagnostics'].append(diagnostic) + candidate_group['pages'].append({ + 'start_row': diagnostic.get('start_row') or 0, + 'returned_rows': diagnostic.get('returned_rows') or diagnostic.get('data_row_count') or 0, + 'rows': result_payload.get('data'), + }) + if diagnostic.get('total_matches') != candidate_group['total_matches']: + candidate_group['total_count_mismatch'] = True + + best_candidate = None + best_score = None + for candidate_group in candidate_groups.values(): + coalesced_result = _coalesce_tabular_generated_output_pages( + candidate_group.get('pages'), + candidate_group.get('total_matches'), + ) + if candidate_group.get('total_count_mismatch'): + coalesced_result['full_result_available'] = False + coalesced_result['validation_error'] = 'Compatible pages reported inconsistent total row counts' + if len(logical_source_identities.get(candidate_group.get('logical_signature'), set())) > 1: + coalesced_result['full_result_available'] = False + coalesced_result['validation_error'] = ( + 'Compatible pages resolved to different source blobs or versions' + ) + score = ( - 1 if diagnostic.get('full_result_available') else 0, - diagnostic.get('returned_rows') or diagnostic.get('data_row_count') or 0, - diagnostic.get('function_rank') or 0, + 1 if coalesced_result.get('full_result_available') else 0, + coalesced_result.get('row_count') or 0, + candidate_group.get('function_rank') or 0, ) if best_score is not None and score <= best_score: continue - result_payload = get_tabular_invocation_result_payload(invocation) best_candidate = { - 'function_name': diagnostic.get('function_name'), - 'filename': result_payload.get('filename'), - 'selected_sheet': result_payload.get('selected_sheet'), - 'rows': result_payload.get('data'), - 'row_count': diagnostic.get('returned_rows') or diagnostic.get('data_row_count'), - 'total_matches': diagnostic.get('total_matches'), - 'full_result_available': diagnostic.get('full_result_available'), - 'diagnostics': diagnostic, + 'function_name': candidate_group.get('function_name'), + 'filename': candidate_group.get('filename'), + 'selected_sheet': candidate_group.get('selected_sheet'), + 'source_parameters': candidate_group.get('source_parameters'), + 'source_descriptor': candidate_group.get('source_descriptor'), + 'source_authorization': candidate_group.get('source_authorization'), + 'rows': coalesced_result.get('rows'), + 'row_count': coalesced_result.get('row_count'), + 'total_matches': candidate_group.get('total_matches'), + 'full_result_available': coalesced_result.get('full_result_available'), + 'validation_error': coalesced_result.get('validation_error'), + 'page_count': len(candidate_group.get('pages') or []), + 'diagnostics': candidate_group.get('diagnostics'), } best_score = score return best_candidate -def _build_tabular_generated_output_batch_prompt(user_question, batch_rows, batch_index, total_batches, source_candidate): +def _build_tabular_generated_output_query_descriptor( + source_candidate, + user_id, + conversation_id, + settings, +): + if not isinstance(source_candidate, dict): + return None + if source_candidate.get('function_name') != 'query_tabular_data': + return None + + source_parameters = source_candidate.get('source_parameters') or {} + query_expression = str(source_parameters.get('query_expression') or '').strip() + source_descriptor = source_candidate.get('source_descriptor') or {} + if ( + not query_expression + or source_descriptor.get('kind') != 'query_tabular_data' + or str(source_descriptor.get('filename') or '') != str(source_candidate.get('filename') or '') + or str(source_descriptor.get('query_expression') or '') != query_expression + ): + return None + validate_tabular_csv_query_expression(query_expression) + + batch_budget = _get_tabular_generated_output_batch_budget(settings) + descriptor = dict(source_descriptor) + descriptor['expected_row_count'] = _safe_int(source_candidate.get('total_matches')) + descriptor['batch_max_rows'] = batch_budget['max_rows'] + descriptor['batch_max_chars'] = batch_budget['max_chars'] + return descriptor + + +def _build_tabular_generated_output_source_authorization(source_candidate): + exact_source_authorization = (source_candidate or {}).get('source_authorization') or {} + if ( + exact_source_authorization.get('source') + and exact_source_authorization.get('container') + and exact_source_authorization.get('blob_path') + ): + return dict(exact_source_authorization) + + source_parameters = (source_candidate or {}).get('source_parameters') or {} + source = str(source_parameters.get('source') or 'chat').strip().lower() + if source not in {'chat', 'workspace', 'group', 'public'}: + source = 'chat' + + authorized_context = dict(getattr(g, 'authorized_chat_context', {}) or {}) + scope_id = None + if source == 'group': + scope_id = str( + source_parameters.get('group_id') + or authorized_context.get('active_group_id') + or '' + ).strip() or None + elif source == 'public': + scope_id = str( + source_parameters.get('public_workspace_id') + or authorized_context.get('active_public_workspace_id') + or '' + ).strip() or None + return { + 'source': source, + 'scope_id': scope_id, + } + + +def _build_failed_tabular_generated_output_metadata(source_candidate, output_format, reason): + normalized_output_format = str(output_format or 'json').strip().lower() or 'json' + row_count = _safe_int((source_candidate or {}).get('total_matches')) + failure_reason = str(reason or 'The exhaustive export could not be prepared.').strip() + return { + 'capability': 'tabular', + 'background_export': True, + 'status': 'failed', + 'status_label': 'Failed', + 'status_tone': 'danger', + 'status_detail': failure_reason, + 'retryable_failure': False, + 'can_resume': False, + 'can_cancel': False, + 'suppress_assistant_table_export': True, + 'file_name': _build_tabular_generated_output_file_name( + (source_candidate or {}).get('filename'), + normalized_output_format, + ), + 'output_format': normalized_output_format, + 'row_count': row_count, + 'processed_rows': 0, + 'source_file_name': (source_candidate or {}).get('filename'), + 'selected_sheet': (source_candidate or {}).get('selected_sheet'), + 'summary': failure_reason, + } + + +def _build_tabular_generated_output_batch_prompt( + user_question, + batch_rows, + batch_index, + total_batches, + source_candidate, + output_schema=None, +): source_file_name = str(source_candidate.get('filename') or 'unknown file').strip() or 'unknown file' selected_sheet = str(source_candidate.get('selected_sheet') or '').strip() batch_rows_json = _dump_tabular_generated_output_json(batch_rows) selected_sheet_line = f"Worksheet: {selected_sheet}\n" if selected_sheet else '' + model_output_schema = [ + field_name + for field_name in (output_schema or []) + if field_name not in {'source_row_number', 'source_row_identity'} + ] + output_schema_line = ( + f'Use exactly these output fields for every object, in this order: ' + f'{json.dumps(model_output_schema, ensure_ascii=False)}.\n' + if model_output_schema + else '' + ) return ( 'Transform the tabular input rows below into structured output for the user.\n\n' f'User instructions:\n{user_question}\n\n' 'Return ONLY a valid JSON array.\n' f'Return exactly {len(batch_rows)} JSON object(s), one per input row, in the same order.\n' + f'{output_schema_line}' + 'Copy __simplechat_source_row_token exactly from each input row into its matching output object. ' + 'Do not include the other fields beginning with __simplechat_source_ in generated objects.\n' 'Do not drop, merge, summarize, or cap rows.\n' 'Input rows may include normalized helper fields such as comment_id, body_text, source_file, attachment_present, attachment_names, and attachment_text. Use those normalized fields when they are present.\n' 'Input rows may include a referenced_documents array containing row-linked evidence from explicitly referenced non-tabular documents. Use that evidence as part of the source row context when it is relevant to the requested output.\n' @@ -4394,6 +4700,25 @@ def _build_tabular_generated_output_system_message(output_metadata): output_format = str(output_metadata.get('output_format') or 'json').upper() file_name = str(output_metadata.get('file_name') or 'generated output').strip() or 'generated output' row_count = _safe_int(output_metadata.get('row_count')) + output_status = str(output_metadata.get('status') or '').strip().lower() + + if output_status == 'failed': + failure_detail = str( + output_metadata.get('status_detail') + or output_metadata.get('summary') + or 'The exhaustive export failed validation.' + ).strip() + return ( + f'The requested exhaustive {output_format} export for {row_count} row(s) failed. ' + f'{failure_detail} Do not claim that a full or partial export is attached, and do not recreate ' + 'the assistant summary table as a CSV. Briefly report the failure and preserve any other requested analysis.' + ) + + if output_status == 'canceled': + return ( + f'The requested exhaustive {output_format} export for {row_count} row(s) was canceled. ' + 'Do not claim that the full export is attached, and do not recreate a partial assistant-table CSV.' + ) if output_metadata.get('background_export'): run_id = str(output_metadata.get('export_run_id') or output_metadata.get('run_id') or '').strip() @@ -4455,13 +4780,14 @@ async def _generate_tabular_structured_output_entries( ): from semantic_kernel.contents.chat_history import ChatHistory as SKChatHistory - rows = [ + normalized_rows = [ _build_tabular_generated_output_input_row( row, source_file_name=source_candidate.get('filename'), ) for row in (source_candidate.get('rows') or []) ] + rows = _prepare_tabular_source_rows(normalized_rows) if not rows: return None @@ -4553,6 +4879,7 @@ async def _generate_tabular_structured_output_entries( ) merged_entries = [] + output_schema = None for batch_index, batch_rows in enumerate(row_batches): batch_number = batch_index + 1 log_event( @@ -4587,6 +4914,7 @@ async def _generate_tabular_structured_output_entries( batch_index, total_batches, source_candidate, + output_schema=output_schema, ) parsed_entries = None @@ -4625,6 +4953,18 @@ async def _generate_tabular_structured_output_entries( if result and result[0].content: parsed_entries = _parse_tabular_generated_json_entries(raw_response_content) parsed_entry_count = len(parsed_entries) if parsed_entries is not None else 0 + validation_error = None + if parsed_entries is not None and parsed_entry_count == len(batch_rows): + try: + parsed_entries, candidate_output_schema = _normalize_generated_batch_entries( + batch_rows, + parsed_entries, + expected_output_schema=output_schema, + ) + output_schema = candidate_output_schema + except ValueError as exc: + validation_error = str(exc) + parsed_entries = None if parsed_entries is None or parsed_entry_count != len(batch_rows): log_event( '[Tabular Generated Output] Structured export batch attempt mismatch', @@ -4636,6 +4976,7 @@ async def _generate_tabular_structured_output_entries( 'attempt_number': attempt_number, 'expected_row_count': len(batch_rows), 'parsed_row_count': parsed_entry_count, + 'validation_error': validation_error, 'response_char_count': len(raw_response_content), 'response_preview': _truncate_tabular_generated_output_response_preview(raw_response_content), }, @@ -4718,6 +5059,172 @@ async def maybe_create_tabular_generated_output( debug_only=True, ) return None + + source_candidate['source_authorization'] = _build_tabular_generated_output_source_authorization( + source_candidate + ) + source_descriptor = None + source_descriptor_error = None + if ( + user_id + and conversation_id + and question_requests_tabular_structured_object_output(user_question) + ): + try: + source_descriptor = _build_tabular_generated_output_query_descriptor( + source_candidate, + user_id, + conversation_id, + settings, + ) + except Exception as exc: + source_descriptor_error = str(exc) + log_event( + '[Tabular Generated Output] Could not build durable source query descriptor', + { + 'conversation_id': conversation_id, + 'source_file_name': source_candidate.get('filename'), + 'function_name': source_candidate.get('function_name'), + 'error': str(exc), + }, + level=logging.WARNING, + ) + if source_descriptor: + source_candidate['source_authorization'] = { + field_name: source_descriptor.get(field_name) + for field_name in ('source', 'scope_id', 'container', 'blob_path') + } + + expected_row_count = _safe_int(source_candidate.get('total_matches')) + source_batch_rows = _safe_int((source_descriptor or {}).get('batch_max_rows')) or 1 + estimated_batch_count = max( + 1, + (expected_row_count + source_batch_rows - 1) // source_batch_rows, + ) + materialized_rows = None + materialized_batches = None + if ( + source_candidate.get('full_result_available') + and _safe_int(source_candidate.get('page_count')) > 1 + ): + materialized_rows = [ + _build_tabular_generated_output_input_row( + row, + source_file_name=source_candidate.get('filename'), + ) + for row in (source_candidate.get('rows') or []) + ] + materialized_batches = _build_tabular_generated_output_row_batches( + materialized_rows, + settings=settings, + ) + threshold_batch_count = len(materialized_batches) if materialized_batches else estimated_batch_count + exceeds_background_threshold = should_queue_tabular_generated_output_background( + expected_row_count, + threshold_batch_count, + settings, + ) + should_queue_materialized_pages = bool( + user_id + and conversation_id + and question_requests_tabular_structured_object_output(user_question) + and source_candidate.get('full_result_available') + and _safe_int(source_candidate.get('page_count')) > 1 + and not exceeds_background_threshold + ) + if should_queue_materialized_pages: + background_run = queue_tabular_generated_output_run( + user_id=user_id, + conversation_id=conversation_id, + user_question=user_question, + source_candidate=source_candidate, + output_format=output_format, + row_batches=materialized_batches, + gpt_model=gpt_model, + settings=settings, + model_context=model_context, + ) + background_metadata = build_background_tabular_generated_output_metadata(background_run) + await emit_tabular_post_processing_thought( + thought_callback, + f"Queued exhaustive {str(output_format or 'json').upper()} export from validated tabular pages", + detail=( + f"run_id={background_metadata.get('export_run_id')}; " + f"rows={expected_row_count}; batches={len(materialized_batches)}; checkpointed=true" + ), + activity=build_tabular_post_processing_activity_payload( + 'tabular.generated_output', + f"Exhaustive {str(output_format or 'json').upper()} export queued", + 'running', + phase='queued', + output_format=output_format, + file_name=source_candidate.get('filename'), + batch_index=0, + batch_count=len(materialized_batches), + ), + ) + return background_metadata + + should_queue_source_backed_run = bool( + source_descriptor + and expected_row_count > 0 + and ( + not source_candidate.get('full_result_available') + or exceeds_background_threshold + ) + ) + if should_queue_source_backed_run: + try: + background_run = queue_tabular_generated_output_run( + user_id=user_id, + conversation_id=conversation_id, + user_question=user_question, + source_candidate=source_candidate, + output_format=output_format, + row_batches=None, + gpt_model=gpt_model, + settings=settings, + model_context=model_context, + source_descriptor=source_descriptor, + ) + except Exception as exc: + log_event( + '[Tabular Generated Output] Durable source-backed export queueing failed', + { + 'conversation_id': conversation_id, + 'source_file_name': source_candidate.get('filename'), + 'row_count': expected_row_count, + 'error': str(exc), + }, + level=logging.ERROR, + exceptionTraceback=True, + ) + return _build_failed_tabular_generated_output_metadata( + source_candidate, + output_format, + 'The exhaustive export could not be queued. No partial CSV was created.', + ) + background_metadata = build_background_tabular_generated_output_metadata(background_run) + await emit_tabular_post_processing_thought( + thought_callback, + f"Queued exhaustive {str(output_format or 'json').upper()} export from the authorized source query", + detail=( + f"run_id={background_metadata.get('export_run_id')}; " + f"rows={expected_row_count}; batches~={estimated_batch_count}; checkpointed=true" + ), + activity=build_tabular_post_processing_activity_payload( + 'tabular.generated_output', + f"Exhaustive {str(output_format or 'json').upper()} export queued", + 'running', + phase='queued', + output_format=output_format, + file_name=source_candidate.get('filename'), + batch_index=0, + batch_count=estimated_batch_count, + ), + ) + return background_metadata + if not source_candidate.get('full_result_available'): log_event( '[Tabular Generated Output] Selected source candidate is incomplete; skipping export', @@ -4725,10 +5232,22 @@ async def maybe_create_tabular_generated_output( 'conversation_id': conversation_id, 'output_format': output_format, 'selected_candidate': source_candidate.get('diagnostics'), + 'validation_error': source_candidate.get('validation_error'), + 'source_replay_available': bool(source_descriptor), }, debug_only=True, ) - return None + failure_detail = source_descriptor_error or source_candidate.get('validation_error') + return _build_failed_tabular_generated_output_metadata( + source_candidate, + output_format, + ( + f'The exhaustive export source could not be validated: {failure_detail}. ' + 'No partial CSV was created.' + if failure_detail + else 'The exhaustive export source was incomplete. No partial CSV was created.' + ), + ) log_event( '[Tabular Generated Output] Selected source candidate', { @@ -4755,7 +5274,11 @@ async def maybe_create_tabular_generated_output( model_context=model_context, ) if output_entries is None: - return None + return _build_failed_tabular_generated_output_metadata( + source_candidate, + output_format, + 'The exhaustive export failed output validation. No partial CSV was created.', + ) if isinstance(output_entries, dict) and output_entries.get('background_export'): return output_entries else: @@ -4835,6 +5358,7 @@ async def maybe_create_tabular_generated_output( ) return { 'capability': 'tabular', + 'suppress_assistant_table_export': True, 'artifact_message_id': upload_result.get('message', {}).get('id'), 'conversation_id': conversation_id, 'storage_scope': 'chat', @@ -19149,10 +19673,29 @@ def tabular_generated_output_run_resume_api(run_id): resume_result = resume_tabular_generated_output_run(user_id, run_id) if not resume_result: return jsonify({'error': 'Tabular generated-output run not found'}), 404 + if resume_result.get('authorization_failed'): + return jsonify(resume_result), 403 if not resume_result.get('success'): return jsonify(resume_result), 409 return jsonify(resume_result) + @bp.route('/api/tabular/generated-output/runs//cancel', methods=['POST']) + @swagger_route(security=get_auth_security()) + @login_required + @user_required + def tabular_generated_output_run_cancel_api(run_id): + """Cancel a generated-output run owned by the current user.""" + user_id = get_current_user_id() + if not user_id: + return jsonify({'error': 'User not authenticated'}), 401 + + cancel_result = cancel_tabular_generated_output_run(user_id, run_id) + if not cancel_result: + return jsonify({'error': 'Tabular generated-output run not found'}), 404 + if not cancel_result.get('success'): + return jsonify(cancel_result), 409 + return jsonify(cancel_result) + @bp.route('/api/chat/stream/reattach/', methods=['GET']) @swagger_route(security=get_auth_security()) @login_required diff --git a/application/single_app/semantic_kernel_plugins/plugin_invocation_logger.py b/application/single_app/semantic_kernel_plugins/plugin_invocation_logger.py index 92dedaf5f..ffdba3a59 100644 --- a/application/single_app/semantic_kernel_plugins/plugin_invocation_logger.py +++ b/application/single_app/semantic_kernel_plugins/plugin_invocation_logger.py @@ -64,6 +64,15 @@ AUTHORIZATION_VALUE_RE = re.compile(r"(?i)\b(Bearer|Basic)\s+[A-Za-z0-9._~+/=-]+") +class PluginInvocationResult(str): + """String tool result with server-only metadata retained in invocation history.""" + + def __new__(cls, value: str, internal_metadata: Optional[Dict[str, Any]] = None): + instance = super().__new__(cls, value) + instance.internal_metadata = dict(internal_metadata or {}) + return instance + + def _normalize_sensitive_key(key: Any) -> str: return re.sub(r"[^a-z0-9]", "", str(key or "").strip().lower()) diff --git a/application/single_app/semantic_kernel_plugins/tabular_processing_plugin.py b/application/single_app/semantic_kernel_plugins/tabular_processing_plugin.py index 99ce381e2..c038cd0ce 100644 --- a/application/single_app/semantic_kernel_plugins/tabular_processing_plugin.py +++ b/application/single_app/semantic_kernel_plugins/tabular_processing_plugin.py @@ -13,15 +13,18 @@ import json import logging import re +import tempfile import warnings import pandas +from azure.core import MatchConditions from flask import g, has_request_context from typing import Annotated, Dict, List, Optional, Set from urllib.parse import urlsplit, urlunsplit from semantic_kernel.functions import kernel_function -from semantic_kernel_plugins.plugin_invocation_logger import plugin_function_logger +from semantic_kernel_plugins.plugin_invocation_logger import PluginInvocationResult, plugin_function_logger from functions_appinsights import log_event from functions_authentication import get_current_user_id +from functions_tabular_csv_query import iter_tabular_csv_query_rows from functions_group import find_group_by_id, get_user_role_in_group from functions_public_workspaces import get_user_visible_public_workspace_ids_from_settings from config import ( @@ -124,6 +127,7 @@ def __init__(self): self._df_cache = {} # Per-instance cache: (container, blob_name, sheet_name) -> DataFrame self._blob_data_cache = {} # Per-instance cache: (container, blob_name) -> raw bytes self._workbook_metadata_cache = {} # Per-instance cache: (container, blob_name) -> workbook metadata + self._blob_version_cache = {} # Per-instance cache: (container, blob_name) -> immutable blob version metadata self._default_sheet_overrides = {} # (container, blob_name) -> default sheet name self._resolved_blob_location_overrides = {} # (source, filename) -> (container, blob_name) @@ -392,11 +396,45 @@ def _download_tabular_blob_bytes(self, container_name: str, blob_name: str) -> b client = self._get_blob_service_client() blob_client = client.get_blob_client(container=container_name, blob=blob_name) - stream = blob_client.download_blob() + blob_version = self._get_tabular_blob_version( + container_name, + blob_name, + refresh=True, + ) + stream = blob_client.download_blob( + etag=blob_version['blob_etag'], + match_condition=MatchConditions.IfNotModified, + ) data = stream.readall() self._blob_data_cache[cache_key] = data return data + def _get_tabular_blob_version(self, container_name: str, blob_name: str, refresh: bool = False) -> dict: + """Return the exact blob version used by this plugin instance.""" + cache_key = (container_name, blob_name) + if not refresh and cache_key in self._blob_version_cache: + return dict(self._blob_version_cache[cache_key]) + + blob_client = self._get_blob_service_client().get_blob_client( + container=container_name, + blob=blob_name, + ) + blob_properties = blob_client.get_blob_properties() + blob_etag = getattr(blob_properties, 'etag', None) + blob_size = getattr(blob_properties, 'size', None) + if isinstance(blob_properties, dict): + blob_etag = blob_etag or blob_properties.get('etag') + blob_size = blob_size if blob_size is not None else blob_properties.get('size') + if not blob_etag: + raise ValueError('Tabular source version could not be determined') + + blob_version = { + 'blob_etag': str(blob_etag), + 'blob_size': int(blob_size or 0), + } + self._blob_version_cache[cache_key] = blob_version + return dict(blob_version) + def _get_excel_engine(self, blob_name: str) -> Optional[str]: """Return the pandas Excel engine for a workbook, or None for CSV files.""" name_lower = blob_name.lower() @@ -3384,6 +3422,169 @@ def _resolve_blob_location_with_fallback(self, user_id: str, conversation_id: st return attempts[0] raise ValueError(f"Could not resolve blob location for {filename}") + def build_generated_export_query_descriptor( + self, + user_id: str, + conversation_id: str, + filename: str, + query_expression: str, + source: str = 'chat', + return_columns: Optional[str] = None, + group_id: Optional[str] = None, + public_workspace_id: Optional[str] = None, + expected_row_count: int = 0, + ) -> dict: + """Resolve an authorized, version-pinned CSV query for durable export replay.""" + container_name, blob_path = self._resolve_blob_location_with_fallback( + user_id, + conversation_id, + filename, + source, + group_id=group_id, + public_workspace_id=public_workspace_id, + ) + return self._build_generated_export_query_descriptor_from_location( + container_name=container_name, + blob_path=blob_path, + filename=filename, + query_expression=query_expression, + return_columns=return_columns, + expected_row_count=expected_row_count, + ) + + def _build_generated_export_query_descriptor_from_location( + self, + container_name: str, + blob_path: str, + filename: str, + query_expression: str, + return_columns: Optional[str] = None, + expected_row_count: int = 0, + blob_version: Optional[dict] = None, + ) -> dict: + """Pin a durable query descriptor to an already-authorized blob location.""" + if not str(blob_path or '').lower().endswith('.csv'): + raise ValueError('Durable source-backed generated exports currently require a CSV source') + + blob_version = dict(blob_version or self._get_tabular_blob_version(container_name, blob_path)) + blob_etag = blob_version.get('blob_etag') + blob_size = blob_version.get('blob_size') + + resolved_source = self._infer_source_from_container(container_name) + blob_parts = [part for part in str(blob_path or '').split('/') if part] + scope_id = blob_parts[0] if resolved_source in {'group', 'public'} and blob_parts else None + return { + 'version': 1, + 'kind': 'query_tabular_data', + 'source': resolved_source, + 'scope_id': scope_id, + 'container': container_name, + 'blob_path': blob_path, + 'blob_etag': str(blob_etag), + 'blob_size': int(blob_size or 0), + 'filename': str(filename or '').strip(), + 'query_expression': str(query_expression or '').strip(), + 'return_columns': return_columns, + 'expected_row_count': max(0, int(expected_row_count or 0)), + } + + def _build_source_authorization_from_location( + self, + container_name: str, + blob_path: str, + blob_version: Optional[dict] = None, + ) -> dict: + """Build exact server-only scope metadata for later worker revalidation.""" + resolved_source = self._infer_source_from_container(container_name) + blob_parts = [part for part in str(blob_path or '').split('/') if part] + scope_id = blob_parts[0] if resolved_source in {'group', 'public'} and blob_parts else None + return { + 'source': resolved_source, + 'scope_id': scope_id, + 'container': container_name, + 'blob_path': blob_path, + 'blob_etag': (blob_version or self._get_tabular_blob_version(container_name, blob_path)).get( + 'blob_etag' + ), + } + + def _query_csv_data_in_bounded_chunks( + self, + container_name: str, + blob_path: str, + filename: str, + query_expression: str, + return_columns: Optional[str], + start_row, + max_rows, + ) -> PluginInvocationResult: + """Execute foreground CSV pagination through the durable replay query engine.""" + start, limit = self._parse_row_page_arguments(start_row, max_rows) + replay_stats = {'used_reviewer_style_fallback': False} + matched_row_count = 0 + page_rows = [] + blob_client = self._get_blob_service_client().get_blob_client( + container=container_name, + blob=blob_path, + ) + blob_version = self._get_tabular_blob_version( + container_name, + blob_path, + refresh=True, + ) + with tempfile.SpooledTemporaryFile(max_size=1024 * 1024, mode='w+b') as source_stream: + blob_client.download_blob( + etag=blob_version['blob_etag'], + match_condition=MatchConditions.IfNotModified, + ).readinto(source_stream) + source_stream.seek(0) + for _, source_row in iter_tabular_csv_query_rows( + csv_stream=source_stream, + query_expression=query_expression, + return_columns=return_columns, + source_chunk_rows=1000, + tabular_plugin=self, + replay_stats=replay_stats, + ): + if start <= matched_row_count < start + limit: + page_rows.append(source_row) + matched_row_count += 1 + + response_payload = { + 'filename': filename, + 'selected_sheet': None, + 'query_expression': query_expression, + 'query_expression_fallback': replay_stats['used_reviewer_style_fallback'], + 'total_matches': matched_row_count, + } + response_payload.update(self._build_tabular_row_page_payload( + page_rows, + matched_row_count, + start_row=start, + max_rows=limit, + return_columns=self._parse_optional_column_list_argument(return_columns), + )) + source_descriptor = self._build_generated_export_query_descriptor_from_location( + container_name=container_name, + blob_path=blob_path, + filename=filename, + query_expression=query_expression, + return_columns=return_columns, + expected_row_count=matched_row_count, + blob_version=blob_version, + ) + return PluginInvocationResult( + json.dumps(response_payload, indent=2, default=str), + internal_metadata={ + 'tabular_generated_export_source': source_descriptor, + 'tabular_source_authorization': self._build_source_authorization_from_location( + container_name, + blob_path, + blob_version=blob_version, + ), + }, + ) + @kernel_function( description=( "List all tabular data files available for a user. Checks workspace documents " @@ -4462,6 +4663,16 @@ def _sync_work(): user_id, conversation_id, filename, source, group_id=group_id, public_workspace_id=public_workspace_id ) + if str(blob_path or '').lower().endswith('.csv'): + return self._query_csv_data_in_bounded_chunks( + container_name=container, + blob_path=blob_path, + filename=filename, + query_expression=query_expression, + return_columns=return_columns, + start_row=start_row, + max_rows=max_rows, + ) # When no explicit sheet_name is given, try cross-sheet query first normalized_sheet = (sheet_name or '').strip() normalized_sheet_idx = None if sheet_index is None else str(sheet_index).strip() @@ -4510,7 +4721,31 @@ def _sync_work(): max_rows=limit, return_columns=parsed_return_columns, )) - return json.dumps(response_payload, indent=2, default=str) + response_json = json.dumps(response_payload, indent=2, default=str) + source_blob_version = self._get_tabular_blob_version(container, blob_path) + internal_metadata = { + 'tabular_source_authorization': self._build_source_authorization_from_location( + container, + blob_path, + blob_version=source_blob_version, + ), + } + if str(blob_path or '').lower().endswith('.csv'): + internal_metadata['tabular_generated_export_source'] = ( + self._build_generated_export_query_descriptor_from_location( + container_name=container, + blob_path=blob_path, + filename=filename, + query_expression=query_expression, + return_columns=return_columns, + expected_row_count=len(result_df), + blob_version=source_blob_version, + ) + ) + return PluginInvocationResult( + response_json, + internal_metadata=internal_metadata, + ) except Exception as e: log_event(f"[TabularProcessingPlugin] Error querying data: {e}", level=logging.WARNING) return json.dumps({"error": f"Query error: {str(e)}. Ensure column names and values are correct."}) diff --git a/application/single_app/static/js/chat/chat-messages.js b/application/single_app/static/js/chat/chat-messages.js index 021d38646..7c5ba84f9 100644 --- a/application/single_app/static/js/chat/chat-messages.js +++ b/application/single_app/static/js/chat/chat-messages.js @@ -3196,7 +3196,12 @@ function renderReplyQuoteHtml(fullMessageObject = null) { const normalizedDocumentId = String(output.document_id || '').trim(); const normalizedExportRunId = String(output.export_run_id || output.run_id || '').trim(); const isBackgroundExport = Boolean(output.background_export) && Boolean(normalizedExportRunId); - if (!normalizedArtifactMessageId && !normalizedDocumentId && !isBackgroundExport) { + const terminalStatus = String(output.status || '').trim().toLowerCase(); + const isTerminalExportStatus = Boolean( + output.suppress_assistant_table_export + && ['failed', 'canceled'].includes(terminalStatus) + ); + if (!normalizedArtifactMessageId && !normalizedDocumentId && !isBackgroundExport && !isTerminalExportStatus) { return null; } @@ -3207,7 +3212,8 @@ function renderReplyQuoteHtml(fullMessageObject = null) { document_id: normalizedDocumentId, export_run_id: normalizedExportRunId, run_id: normalizedExportRunId, - background_export: isBackgroundExport, + background_export: isBackgroundExport || isTerminalExportStatus, + suppress_assistant_table_export: Boolean(output.suppress_assistant_table_export), }; } @@ -3393,6 +3399,36 @@ function renderReplyQuoteHtml(fullMessageObject = null) { } } + function canCancelBackgroundGeneratedOutput(outputMetadata) { + return Boolean(outputMetadata?.background_export && outputMetadata?.can_cancel); + } + + function setBackgroundGeneratedOutputCancelButtonLabel(cancelButton, label) { + if (!(cancelButton instanceof HTMLElement)) { + return; + } + + const icon = document.createElement('i'); + icon.className = 'bi bi-x-circle me-1'; + icon.setAttribute('aria-hidden', 'true'); + const labelText = document.createElement('span'); + labelText.textContent = label; + cancelButton.replaceChildren(icon, labelText); + } + + function updateBackgroundGeneratedOutputCancelButton(cancelButton, outputMetadata) { + if (!(cancelButton instanceof HTMLElement)) { + return; + } + + const canCancel = canCancelBackgroundGeneratedOutput(outputMetadata); + cancelButton.classList.toggle('d-none', !canCancel); + cancelButton.disabled = !canCancel; + if (cancelButton.dataset.busy !== 'true') { + setBackgroundGeneratedOutputCancelButtonLabel(cancelButton, 'Cancel'); + } + } + function formatGeneratedTabularPreviewValue(value, maxLength = 120) { let formattedValue = ''; @@ -4227,6 +4263,7 @@ function renderReplyQuoteHtml(fullMessageObject = null) { updateBackgroundGeneratedOutputStatusCard(statusElements, outputMetadata); updateBackgroundGeneratedOutputContinueButton(statusElements.continueButton, outputMetadata); + updateBackgroundGeneratedOutputCancelButton(statusElements.cancelButton, outputMetadata); } catch (error) { if (statusElements.detailText) { statusElements.detailText.textContent = error.message || 'Could not refresh export progress.'; @@ -4298,6 +4335,53 @@ function renderReplyQuoteHtml(fullMessageObject = null) { } } + async function cancelBackgroundGeneratedOutputRun(outputMetadata, card, statusElements = {}, cancelButton = null) { + const runId = String(outputMetadata?.export_run_id || outputMetadata?.run_id || '').trim(); + if (!runId || !(card instanceof HTMLElement) || !document.body.contains(card)) { + return; + } + + if (cancelButton) { + cancelButton.dataset.busy = 'true'; + cancelButton.disabled = true; + setBackgroundGeneratedOutputCancelButtonLabel(cancelButton, 'Canceling...'); + } + + try { + const response = await fetch(`/api/tabular/generated-output/runs/${encodeURIComponent(runId)}/cancel`, { + method: 'POST', + headers: { + 'Accept': 'application/json', + }, + }); + const responseData = await response.json().catch(() => ({})); + if (!response.ok) { + throw new Error(responseData?.message || responseData?.error || `Server responded with status ${response.status}`); + } + + const runStatus = responseData?.run || {}; + Object.assign(outputMetadata, runStatus, { + export_run_id: runStatus.run_id || runId, + run_id: runStatus.run_id || runId, + background_export: true, + }); + updateBackgroundGeneratedOutputStatusCard(statusElements, outputMetadata); + updateBackgroundGeneratedOutputContinueButton(statusElements.continueButton, outputMetadata); + updateBackgroundGeneratedOutputCancelButton(cancelButton, outputMetadata); + showToast(responseData?.message || 'Background export canceled.', 'success'); + } catch (error) { + if (statusElements.detailText) { + statusElements.detailText.textContent = error.message || 'Could not cancel background export.'; + } + showToast(error.message || 'Could not cancel background export.', 'danger'); + } finally { + if (cancelButton) { + delete cancelButton.dataset.busy; + updateBackgroundGeneratedOutputCancelButton(cancelButton, outputMetadata); + } + } + } + function shouldPollBackgroundGeneratedOutput(outputMetadata) { if (!outputMetadata?.background_export) { return false; @@ -4451,6 +4535,11 @@ function renderReplyQuoteHtml(fullMessageObject = null) { actions.className = 'd-flex flex-wrap gap-2 mt-3'; if (outputMetadata?.background_export) { + const backgroundRunId = String(outputMetadata?.export_run_id || outputMetadata?.run_id || '').trim(); + if (!backgroundRunId) { + return card; + } + const continueButton = document.createElement('button'); continueButton.type = 'button'; continueButton.className = 'btn btn-sm btn-outline-primary generated-tabular-continue-btn d-none'; @@ -4464,18 +4553,20 @@ function renderReplyQuoteHtml(fullMessageObject = null) { updateBackgroundGeneratedOutputContinueButton(continueButton, outputMetadata); actions.appendChild(continueButton); - const refreshStatusButton = document.createElement('button'); - refreshStatusButton.type = 'button'; - refreshStatusButton.className = 'btn btn-sm btn-outline-secondary generated-tabular-refresh-status-btn'; - refreshStatusButton.textContent = 'Refresh Status'; - refreshStatusButton.addEventListener('click', async () => { - refreshStatusButton.disabled = true; - refreshStatusButton.textContent = 'Refreshing...'; - await refreshBackgroundGeneratedOutputStatus(outputMetadata, card, backgroundStatusElements || {}); - refreshStatusButton.disabled = false; - refreshStatusButton.textContent = 'Refresh Status'; + const cancelButton = document.createElement('button'); + cancelButton.type = 'button'; + cancelButton.className = 'btn btn-sm btn-outline-danger generated-tabular-cancel-btn d-none'; + cancelButton.setAttribute('aria-label', 'Cancel background export'); + setBackgroundGeneratedOutputCancelButtonLabel(cancelButton, 'Cancel'); + cancelButton.addEventListener('click', async () => { + await cancelBackgroundGeneratedOutputRun(outputMetadata, card, backgroundStatusElements || {}, cancelButton); }); - actions.appendChild(refreshStatusButton); + if (backgroundStatusElements) { + backgroundStatusElements.cancelButton = cancelButton; + } + updateBackgroundGeneratedOutputCancelButton(cancelButton, outputMetadata); + actions.appendChild(cancelButton); + card.appendChild(actions); scheduleBackgroundGeneratedOutputStatusPolling(outputMetadata, card, backgroundStatusElements || {}); return card; diff --git a/docs/explanation/fixes/TABULAR_ROW_ORCHESTRATION_REMEDIATION_PLAN.md b/docs/explanation/fixes/TABULAR_ROW_ORCHESTRATION_REMEDIATION_PLAN.md new file mode 100644 index 000000000..a9d99a2f3 --- /dev/null +++ b/docs/explanation/fixes/TABULAR_ROW_ORCHESTRATION_REMEDIATION_PLAN.md @@ -0,0 +1,192 @@ +# Tabular Row Orchestration Remediation Plan + +Fixed in version: **0.250.060** + +UI follow-up implemented in version: **0.250.061** + +Related issue: **microsoft/simplechat#1031** + +## Issue Description + +Per-row tabular analysis could produce a complete export for small files but fail once a query result was split across tool pages. The generated-output selector evaluated each page independently, rejected every partial page, and could then allow the generic assistant-table exporter to save a small summary table as though it were the requested exhaustive CSV. + +The durable export runner also staged all input batches in one JSON blob and loaded every output checkpoint into one Python list during finalization. Those two operations prevented a defensible bounded-memory guarantee for 3,000- and 30,000-row exports. + +## Root Cause Analysis + +- Compatible `query_tabular_data` pages were ranked independently instead of being validated as ordered intervals from one query. +- Large runs could only be queued after all source rows had already been materialized by the chat request. +- Input staging used one aggregate `input_batches.json` payload. +- Final CSV/JSON assembly consolidated all output rows in memory. +- Generated batches had no authoritative source ordinal, persisted output schema, or explicit schema-drift validation. +- Background execution trusted the stored run identity without revalidating current conversation ownership and workspace access. +- The runner had a canceled status constant but no cancellation transition or user control. + +## Version Implemented + +Fixed in version: **0.250.060**. + +`application/single_app/config.py` was updated from `0.250.059` to `0.250.060`. + +## Technical Details + +### Architecture + +The existing durable generated-export subsystem remains the only background execution path. Issue #1031 extends it with two input modes: + +1. **Direct rows** for small, complete tool results. Rows are assigned canonical source ordinals and identities before per-batch staging. +2. **Authorized source queries** for incomplete, multi-page, or threshold-large CSV queries. The request resolves an exact blob location and ETag, and the worker revalidates access before replaying the query in bounded CSV chunks. + +Both modes converge on the same model batching, retry, checkpoint, progress, cancellation, and final artifact lifecycle. + +### Source Contract + +Every input row receives: + +- `__simplechat_source_row_number`: a canonical one-based ordinal. +- `__simplechat_source_row_identity`: a stable source identifier selected from fields such as Case ID, record ID, comment ID, submission ID, or ID, with the ordinal as fallback. +- `__simplechat_source_row_token`: a deterministic opaque token that the model must echo for the matching row. + +Every output row receives authoritative `source_row_number` and `source_row_identity` fields. Model-supplied values for those fields are ignored. + +The echoed opaque tokens must match the exact ordered input sequence. A same-length response with swapped rows therefore fails before source identities are attached. + +The first successful generated batch establishes the output schema. Later batches must contain exactly the same field set, and finalization validates schema, source ordinal continuity, and total row count before publication. + +### Paginated Query Handling + +Compatible tabular invocations are grouped by plugin, function, file, worksheet, query, projection, and authorized source parameters while excluding pagination controls. Their intervals are sorted and validated. + +The grouping key also includes the server-resolved container, blob path, workspace scope, and ETag. Each page download is conditionally pinned to that ETag. Pages from different blob paths or versions fail explicitly instead of being coalesced. + +- Contiguous pages are coalesced in source order. +- Gaps, overlaps, inconsistent totals, and declared/actual page-size mismatches remain incomplete and fail closed. +- Replayable multi-page and incomplete structured queries are queued from an authorized source descriptor instead of sending all rows through model context. + +### Bounded Source Staging + +Source-backed runs persist the resolved source scope, blob path, ETag, expected match count, query expression, projection, and batch limits. The descriptor is never returned in public run status. + +The exact authorized blob path and ETag are captured as server-only invocation metadata on the original query result. Descriptor creation never resolves the file again by filename. + +At each worker start or resume: + +- Personal conversation ownership is revalidated. +- Personal, group, or public workspace access is revalidated against current authorization state. +- The source ETag is compared with the queued version. +- Foreground CSV pagination and durable replay use the same bounded query engine, numeric-column inference, row-local expression validator, projection, and hidden-reference preservation. +- The row-local expression validator parses a strict grammar of column references, comparisons, boolean operators, arithmetic, constants, and list/tuple membership. Function calls, attributes, subscripting, external variables, aggregations, and other cross-row operations are rejected before queueing. +- Each complete input batch is written to its own blob. +- The physical source row reached, staged batch count, and staged output-row count are checkpointed for resume. Resumed CSV reads use a callable skip predicate, keeping skip state constant-size instead of allocating one entry per skipped row. + +A changed source, authorization loss, or result-count mismatch fails explicitly before model processing or final artifact publication. + +Authorization runs immediately after claim and before legacy migration, source staging, or checkpoint reads. Manual Continue also reauthorizes before its ETag transition; revoked access returns a forbidden response without submitting work. + +### Model Output Checkpoints + +The runner processes bounded concurrent windows while forcing the first batch to run alone and establish the schema. Successful batches are checkpointed independently. Existing output checkpoints are reused after transient failures or worker restarts. + +Malformed JSON, row-count mismatch, missing fields, unexpected fields, or schema drift fails the batch before it advances contiguous progress. + +Each checkpoint also stores a compact bounded summary containing field completeness and limited scalar value counts. These summaries are merged after validation to produce the completed artifact card's compact overall analysis without putting all output rows back into model context. + +### Atomic Finalization + +Final CSV and JSON assembly reads one ordered checkpoint at a time into a disk-backed spooled stream. + +- CSV uses `csv.DictWriter` for quoting and encoding. +- JSON is emitted as one valid ordered array. +- Every row is revalidated for schema and contiguous source ordinal. +- The expected row count must match exactly. +- The configured generated-artifact size limit is enforced before upload. +- A unique final blob is uploaded before its chat artifact message is published. + +A validation or upload failure leaves no user-visible completed artifact. + +### Progress, Retry, and Cancellation + +Existing retry classification, scheduler recovery, leases, progress status, and manual Continue behavior are preserved. The run status now also exposes `can_cancel`. + +Every worker claim increments a lease generation. Cosmos state writes are ETag-conditional, stale workers stop on holder/generation mismatch, and generated checkpoint blobs use create-only first-writer-wins semantics for current contracts. + +Scheduler status scans stream runs oldest-first, evaluate the real due/stale/retryable predicate, and only then apply the configured candidate limit. Ineligible rows at the front of a status partition therefore cannot starve later recoverable runs. + +Users can cancel queued, running, retryable, or failed runs from the generated-output card. Workers check the durable canceled state at source and model checkpoint boundaries and immediately before final publication. Canceled runs retain their checkpoint summary but cannot resume or attach a final artifact. + +Status polling is automatic. Version **0.250.061** removed the redundant manual Refresh Status action so running cards show only Cancel; Continue appears only for runs that can genuinely resume. + +Final artifact message and blob identities are deterministic per run. A retry after a partial publication reconciles the same artifact instead of creating a duplicate visible file. Cancellation closes before the fenced publication phase begins, and authorization is revalidated again immediately before upload. + +Runs queued by the pre-`0.250.060` contract are migrated once: aggregate inputs become deterministic per-batch inputs, progress resets, and legacy outputs are regenerated under token/schema validation. + +### Assistant-Table Fallback + +A queued source-backed run returns generated tabular output metadata immediately. Because that metadata identifies a CSV export even while queued, running, failed, or canceled, the generic assistant-table exporter cannot save a partial summary table as the requested exhaustive deliverable. + +All exhaustive outputs carry a format-independent `suppress_assistant_table_export` contract. Terminal failures are preserved through server and browser normalization even when queue creation never produced a run ID, so failed JSON and CSV requests remain visible and cannot fall through to a summary-table CSV. + +## Files Modified + +- `application/single_app/config.py` +- `application/single_app/functions_simplechat_operations.py` +- `application/single_app/functions_tabular_csv_query.py` +- `application/single_app/functions_tabular_generated_exports.py` +- `application/single_app/route_backend_chats.py` +- `application/single_app/semantic_kernel_plugins/plugin_invocation_logger.py` +- `application/single_app/semantic_kernel_plugins/tabular_processing_plugin.py` +- `application/single_app/static/js/chat/chat-messages.js` +- `functional_tests/test_tabular_background_generated_exports.py` +- `functional_tests/test_tabular_large_result_pagination.py` +- `functional_tests/test_tabular_row_orchestration_scale.py` +- `functional_tests/test_assistant_table_csv_artifact.py` +- `ui_tests/test_chat_background_generated_export_status.py` +- `docs/explanation/release_notes.md` + +## Testing Approach + +Focused functional coverage validates: + +- Direct 10-row source identity and stable schema behavior. +- Coalescing the 300-row `94 + 95 + 94 + 17` page sequence into ordered rows from `SC-2001` through `SC-2300`. +- Explicit gap and schema-drift rejection. +- Exact opaque-token rejection for swapped model rows. +- Mixed source-path/ETag page rejection. +- Bounded 30,000-row CSV source scanning and resume from physical row 15,000. +- Real plugin CSV pagination through the shared bounded engine without the whole-DataFrame reader. +- Bounded 30,000-row final CSV assembly across 600 checkpoints. +- Source ordinal gap rejection before publication. +- Current personal, group, public workspace, and conversation authorization at worker execution. +- Idempotent durable cancellation. +- ETag/lease-generation fencing for stale workers and deterministic legacy-run migration. +- Eligibility-before-limit scheduler coverage beyond six ineligible rows. +- Retry-idempotent final artifact publication. +- Compact post-run analysis from 600 batch summaries. +- Source-backed routing and assistant-table fallback suppression. + +Existing background lifecycle, tabular pagination, assistant-table, Python compile, JavaScript syntax, and Flask route-policy tests are also run. + +The authenticated Playwright cancellation workflow is included in `ui_tests/test_chat_background_generated_export_status.py`; it requires `SIMPLECHAT_UI_BASE_URL` and `SIMPLECHAT_UI_STORAGE_STATE`. + +## Impact Analysis + +### Before + +- A paginated 300-row query could be rejected as incomplete even when all pages were present. +- A small assistant summary table could be saved as a misleading CSV. +- Source and final output materialization scaled with the total row count. +- Worker execution did not revalidate current source authorization. + +### After + +- Compatible pages are treated as one validated ordered result. +- Multi-page and large exhaustive transforms run through durable authorized source replay. +- Input and output memory are bounded by source chunks, model batches, and a disk-backed final stream. +- Completed rows survive model interruptions and worker restarts. +- The final artifact appears only after count, order, schema, source version, authorization, encoding, and size validation. +- Progress, retry, Continue, Cancel, failure details, and compact final analysis remain visible in the chat card. + +## Known Limitations + +- Source-backed replay currently targets CSV `query_tabular_data` runs. Complete small workbook results continue to use the direct bounded-batch path. +- Query replay supports row-wise pandas query semantics. Operations that require cross-row aggregation should continue using the dedicated aggregate tabular tools rather than per-row orchestration. diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index 676d03a75..5a25204aa 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,25 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](/explanation/features/) and [Fixes by Version](/explanation/fixes/). +### **(v0.250.061)** + +#### User Interface Enhancements + +* **Automatic Background Export Status Updates** + * Removed the redundant Refresh Status button from generated tabular export cards because progress already updates automatically. + * Running exports now present only the relevant Cancel action, while Continue remains available only when a stalled or retryable run can actually resume. + * (Ref: microsoft/simplechat#1031, `chat-messages.js`, background generated export status cards) + +### **(v0.250.060)** + +#### Bug Fixes + +* **Scalable Per-Row Tabular Analysis and Exports** + * Fixed exhaustive CSV/JSON generation across paginated tabular results by validating compatible pages as one ordered source and preserving one authoritative source identity per output row. + * Added authorized, ETag-pinned CSV query replay with bounded source/input/output windows, resumable checkpoints, stable schema enforcement, streamed atomic finalization, cancellation, and compact checkpoint-derived completion summaries for 30,000+ rows. + * Prevented the generic assistant-table fallback from saving a partial summary CSV while an exhaustive export has an explicit durable status. + * (Ref: microsoft/simplechat#1031, `functions_tabular_generated_exports.py`, `route_backend_chats.py`, `TABULAR_ROW_ORCHESTRATION_REMEDIATION_PLAN.md`) + ### **(v0.250.059)** #### New Features diff --git a/functional_tests/test_assistant_table_csv_artifact.py b/functional_tests/test_assistant_table_csv_artifact.py index 5b4ba9e50..34e5c1e6d 100644 --- a/functional_tests/test_assistant_table_csv_artifact.py +++ b/functional_tests/test_assistant_table_csv_artifact.py @@ -2,8 +2,8 @@ #!/usr/bin/env python3 """ Functional test for assistant-rendered table CSV artifacts. -Version: 0.241.051 -Implemented in: 0.241.050 +Version: 0.250.061 +Implemented in: 0.241.050; exhaustive-export suppression in 0.250.060 This test ensures that explicit table-format requests with assistant-rendered tables and natural CSV/table conversion requests are converted into @@ -21,7 +21,7 @@ APP_DIR = ROOT / 'application' / 'single_app' CONFIG_FILE = APP_DIR / 'config.py' CHAT_ROUTE_FILE = APP_DIR / 'route_backend_chats.py' -EXPECTED_VERSION = '0.241.051' +EXPECTED_VERSION = '0.250.061' sys.path.append(str(APP_DIR)) diff --git a/functional_tests/test_tabular_background_generated_exports.py b/functional_tests/test_tabular_background_generated_exports.py index 41c87ffc9..e5b99a581 100644 --- a/functional_tests/test_tabular_background_generated_exports.py +++ b/functional_tests/test_tabular_background_generated_exports.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """ Functional test for durable tabular generated-output background exports. -Version: 0.241.186 +Version: 0.250.061 Implemented in: 0.241.060 This test ensures that large tabular structured exports are wired through the @@ -63,6 +63,7 @@ def test_export_runner_module(): 'build_background_tabular_generated_output_metadata', 'process_tabular_generated_output_run', 'resume_tabular_generated_output_run', + 'cancel_tabular_generated_output_run', 'check_due_tabular_generated_output_runs_once', '_is_due_queued_retry_run', '_is_stale_queued_run', @@ -74,9 +75,19 @@ def test_export_runner_module(): source_text = read_text(EXPORT_MODULE) assert_contains(source_text, "STATUS_QUEUED = 'queued'", 'queued status constant') - assert_contains(source_text, 'input_batches.json', 'single staged input-batches blob') + assert_contains(source_text, 'input/batch_', 'per-batch staged input blobs') assert_contains(source_text, 'output/batch_', 'per-batch output checkpoint blobs') - assert_contains(source_text, 'upload_generated_analysis_artifact_for_user', 'background-safe artifact upload') + assert_contains(source_text, 'upload_generated_analysis_artifact_stream_for_user', 'bounded-memory artifact upload') + assert_contains(source_text, '_write_ordered_output_stream', 'ordered streaming finalization') + assert_contains(source_text, '_stage_tabular_generated_output_source', 'bounded source-query staging') + assert_contains(source_text, '_authorize_tabular_export_run_execution', 'worker-boundary authorization') + assert_contains(source_text, '_migrate_legacy_tabular_export_run', 'legacy run contract migration') + assert_contains(source_text, 'TABULAR_EXPORT_CONTRACT_VERSION = 2', 'versioned row orchestration contract') + assert_contains(source_text, 'lease_generation', 'worker fencing generation') + assert_contains(source_text, '_replace_claimed_run', 'ETag-fenced worker persistence') + assert_contains(source_text, 'TABULAR_EXPORT_INPUT_ROW_TOKEN_FIELD', 'opaque row binding token') + assert_contains(source_text, '_build_generated_batch_summary', 'per-batch compact summaries') + assert_contains(source_text, '_build_compact_post_run_summary', 'checkpoint-derived post-run summary') assert_contains(source_text, "'generated_artifact': generated_artifact", 'completed artifact status payload') assert_contains(source_text, '_mark_run_retryable', 'retryable transient failure requeue') assert_contains(source_text, 'transient_failure_count', 'bounded transient failure counter') @@ -90,6 +101,8 @@ def test_export_runner_module(): assert_contains(source_text, '_query_scheduler_candidates_by_status', 'simple scheduler status query helper') assert_contains(source_text, '_scheduler_candidate_reason', 'Python-side scheduler due filtering') assert_contains(source_text, 'FROM c WHERE c.type = @type AND c.status = @status', 'Cosmos-safe scheduler query shape') + assert_contains(source_text, 'ORDER BY c.updated_at ASC', 'oldest-first scheduler ordering') + assert_contains(source_text, 'if len(eligible_candidates) >= per_status_limit', 'eligibility-before-limit scheduler scan') assert_contains(source_text, 'active_processing_seconds', 'active-time ETA accounting') assert_contains(source_text, 'or _is_due_queued_retry_run(run)', 'queued retry-due manual resume eligibility') assert_contains(source_text, 'or _is_stale_queued_run(run, settings or {})', 'stale queued manual resume eligibility') @@ -97,12 +110,17 @@ def test_export_runner_module(): assert_contains(source_text, "'retry_due': status_detail.get('retry_due')", 'retry-due public status payload') assert_contains(source_text, 'Manual resume queued', 'manual checkpoint resume message') assert_contains(source_text, 'manual_resume_count', 'manual resume counter') + assert_contains(source_text, 'can_cancel', 'public cancellation capability') assert_contains(source_text, 'status_detail', 'safe status detail payload') assert_contains(source_text, 'checkpoint_summary', 'checkpoint summary payload') assert_contains(source_text, 'waiting_for_retry', 'scheduled retry status payload') assert_contains(source_text, 'retry_delay_seconds', 'retry delay status payload') assert_contains(source_text, 'Background scheduler scan result', 'scheduler scan diagnostics') + simplechat_operations_source = read_text(APP_ROOT / 'functions_simplechat_operations.py') + assert_contains(simplechat_operations_source, 'artifact_idempotency_key', 'idempotent artifact key') + assert_contains(simplechat_operations_source, 'uuid.uuid5', 'deterministic artifact message identity') + def test_background_runner_bounded_batch_concurrency(): """Validate Phase 4 bounded model-batch concurrency in the background runner.""" @@ -138,7 +156,9 @@ def test_chat_route_wires_background_exports(): assert_contains(source_text, 'build_background_tabular_generated_output_metadata', 'background metadata handoff') assert_contains(source_text, "'/api/tabular/generated-output/runs/'", 'run status API route') assert_contains(source_text, "'/api/tabular/generated-output/runs//resume'", 'run resume API route') + assert_contains(source_text, "'/api/tabular/generated-output/runs//cancel'", 'run cancel API route') assert_contains(source_text, 'resume_tabular_generated_output_run', 'manual resume route helper') + assert_contains(source_text, 'cancel_tabular_generated_output_run', 'cancel route helper') assert_contains(source_text, '@swagger_route(security=get_auth_security())', 'secured status route decorator') assert_contains(source_text, "output_metadata.get('background_export')", 'background assistant handoff message') @@ -185,13 +205,17 @@ def test_chat_ui_renders_and_polls_background_exports(): assert_contains(source_text, 'refreshBackgroundGeneratedOutputStatus', 'status refresh function') assert_contains(source_text, 'continueBackgroundGeneratedOutputRun', 'manual continue function') assert_contains(source_text, 'generated-tabular-continue-btn', 'manual continue button') + assert_contains(source_text, 'generated-tabular-cancel-btn', 'cancel button') assert_contains(source_text, '/resume', 'manual resume endpoint call') + assert_contains(source_text, '/cancel', 'cancel endpoint call') assert_contains(source_text, 'formatGeneratedOutputTimestamp', 'localized status timestamps') assert_contains(source_text, 'formatGeneratedOutputDuration', 'readable retry and ETA durations') assert_contains(source_text, 'shouldPollBackgroundGeneratedOutput', 'retry-aware polling guard') assert_contains(source_text, 'status_detail', 'safe status detail rendering') assert_contains(source_text, '/api/tabular/generated-output/runs/', 'status polling endpoint') assert_contains(source_text, 'textContent', 'safe text rendering boundary') + if 'generated-tabular-refresh-status-btn' in source_text or 'Refresh Status' in source_text: + raise AssertionError('Background export cards must rely on automatic polling without a manual refresh button') def main(): diff --git a/functional_tests/test_tabular_large_result_pagination.py b/functional_tests/test_tabular_large_result_pagination.py index bf3d73e67..630093ec8 100644 --- a/functional_tests/test_tabular_large_result_pagination.py +++ b/functional_tests/test_tabular_large_result_pagination.py @@ -2,8 +2,8 @@ # test_tabular_large_result_pagination.py """ Functional test for tabular SK large-result pagination and output trimming. -Version: 0.242.072 -Implemented in: 0.242.067 +Version: 0.250.061 +Implemented in: 0.242.067; bounded CSV query path in 0.250.060 This test ensures row-returning tabular processing tools support start_row/max_rows pagination, avoid skipped rows after auto-trimming oversized output, honor @@ -38,6 +38,37 @@ TabularProcessingPlugin = PLUGIN_MODULE.TabularProcessingPlugin +class MockCsvBlobClient: + """Minimal blob client for bounded CSV query tests.""" + + def __init__(self, content): + self.content = content + + def download_blob(self, etag=None, match_condition=None): + assert etag == 'etag-csv-300' + assert match_condition is not None + content = self.content + + class Downloader: + def readinto(self, stream): + return stream.write(content) + + return Downloader() + + def get_blob_properties(self): + return {'etag': 'etag-csv-300', 'size': len(self.content)} + + +class MockCsvBlobServiceClient: + def __init__(self, content): + self.blob_client = MockCsvBlobClient(content) + + def get_blob_client(self, container, blob): + assert container == 'mock-container' + assert blob == 'nested/version-7/large-results.csv' + return self.blob_client + + def build_workbook_plugin(workbook_frames): """Create a TabularProcessingPlugin backed by in-memory workbook frames.""" plugin = TabularProcessingPlugin() @@ -53,6 +84,10 @@ def build_workbook_plugin(workbook_frames): plugin._resolve_blob_location_with_fallback = lambda *args, **kwargs: (container_name, blob_name) plugin._get_workbook_metadata = lambda *args, **kwargs: workbook_metadata.copy() + plugin._blob_version_cache[(container_name, blob_name)] = { + 'blob_etag': 'etag-workbook-test', + 'blob_size': 0, + } def read_dataframe(container, blob, sheet_name=None, sheet_index=None, require_explicit_sheet=False): selected_sheet, _ = plugin._resolve_sheet_selection( @@ -310,6 +345,56 @@ def test_query_tabular_data_supports_return_columns_and_pagination(): return False +def test_query_tabular_csv_uses_bounded_shared_engine_and_exact_descriptor(): + """Verify CSV queries bypass whole-DataFrame loading and pin the exact analyzed blob.""" + print('🔍 Testing bounded CSV query pagination and source identity...') + + try: + csv_content = ('Case ID,Score,Payload\n' + ''.join( + f'SC-{2001 + row_index},{row_index},payload-{row_index}\n' + for row_index in range(300) + )).encode('utf-8') + plugin = TabularProcessingPlugin() + plugin._resolve_blob_location_with_fallback = lambda *args, **kwargs: ( + 'mock-container', + 'nested/version-7/large-results.csv', + ) + plugin._get_blob_service_client = lambda: MockCsvBlobServiceClient(csv_content) + plugin._read_tabular_blob_to_dataframe = lambda *args, **kwargs: (_ for _ in ()).throw( + AssertionError('CSV query must not use the whole-DataFrame reader') + ) + + result = asyncio.run(plugin.query_tabular_data( + user_id='test-user', + conversation_id='test-conversation', + filename='large-results.csv', + query_expression='Score >= 0', + return_columns='Case ID,Score', + source='chat', + start_row='94', + max_rows='95', + )) + payload = json.loads(result) + descriptor = result.internal_metadata['tabular_generated_export_source'] + + assert payload['total_matches'] == 300, payload + assert payload['returned_rows'] == 95, payload + assert payload['data'][0]['Case ID'] == 'SC-2095', payload + assert payload['data'][-1]['Case ID'] == 'SC-2189', payload + assert descriptor['container'] == 'mock-container', descriptor + assert descriptor['blob_path'] == 'nested/version-7/large-results.csv', descriptor + assert descriptor['blob_etag'] == 'etag-csv-300', descriptor + assert descriptor['expected_row_count'] == 300, descriptor + + print('✅ Bounded CSV query pagination and source identity passed') + return True + except Exception as exc: + print(f'❌ Test failed: {exc}') + import traceback + traceback.print_exc() + return False + + if __name__ == '__main__': tests = [ test_filter_rows_paginates_without_skipping_after_row_trim, @@ -317,6 +402,7 @@ def test_query_tabular_data_supports_return_columns_and_pagination(): test_cross_sheet_filter_rows_paginates_across_sheet_boundary, test_search_rows_preserves_attachment_references_with_return_columns, test_query_tabular_data_supports_return_columns_and_pagination, + test_query_tabular_csv_uses_bounded_shared_engine_and_exact_descriptor, ] results = [] diff --git a/functional_tests/test_tabular_row_orchestration_scale.py b/functional_tests/test_tabular_row_orchestration_scale.py new file mode 100644 index 000000000..9ec4c117a --- /dev/null +++ b/functional_tests/test_tabular_row_orchestration_scale.py @@ -0,0 +1,1224 @@ +# test_tabular_row_orchestration_scale.py +""" +Functional test for scalable per-row tabular orchestration. +Version: 0.250.061 +Implemented in: 0.250.060 + +This test ensures generated exports preserve source identity and row order while +enforcing one stable output schema across independently generated batches. +""" + +import ast +import csv +import io +import importlib.util +import json +import logging +import os +import re +import sys +import uuid +from datetime import datetime, timezone +from pathlib import Path +from types import SimpleNamespace +from collections import Counter +from typing import Any, Dict, Optional + + +REPO_ROOT = Path(__file__).resolve().parents[1] +EXPORT_MODULE = REPO_ROOT / 'application' / 'single_app' / 'functions_tabular_generated_exports.py' +CHAT_ROUTE = REPO_ROOT / 'application' / 'single_app' / 'route_backend_chats.py' +SIMPLECHAT_OPERATIONS = REPO_ROOT / 'application' / 'single_app' / 'functions_simplechat_operations.py' +CSV_QUERY_MODULE = REPO_ROOT / 'application' / 'single_app' / 'functions_tabular_csv_query.py' +CONTRACT_FUNCTIONS = { + '_normalize_source_identity_label', + '_select_source_row_identity', + '_prepare_tabular_source_rows', + '_normalize_generated_batch_entries', +} +CONTRACT_CONSTANTS = { + 'TABULAR_EXPORT_INPUT_ROW_NUMBER_FIELD', + 'TABULAR_EXPORT_INPUT_ROW_IDENTITY_FIELD', + 'TABULAR_EXPORT_INPUT_ROW_TOKEN_FIELD', + 'TABULAR_EXPORT_OUTPUT_ROW_NUMBER_FIELD', + 'TABULAR_EXPORT_OUTPUT_ROW_IDENTITY_FIELD', +} +CANDIDATE_FUNCTIONS = { + '_build_tabular_generated_output_candidate_diagnostic', + '_build_tabular_generated_output_source_signature', + '_coalesce_tabular_generated_output_pages', + '_build_tabular_generated_output_source_candidate', +} +STREAM_FUNCTIONS = { + '_serialize_generated_output_value', + '_write_ordered_output_stream', +} +SOURCE_READER_FUNCTIONS = { + 'detect_tabular_csv_numeric_columns', + 'iter_tabular_csv_query_rows', + 'validate_tabular_csv_query_expression', +} +AUTHORIZATION_FUNCTIONS = {'_authorize_tabular_export_run_execution'} +CANCELLATION_FUNCTIONS = { + '_can_cancel_run', + 'cancel_tabular_generated_output_run', +} +SUMMARY_FUNCTIONS = { + '_build_generated_batch_summary', + '_build_compact_post_run_summary', +} +FENCING_FUNCTIONS = { + '_raise_if_tabular_export_canceled', + '_replace_claimed_run', +} +LEGACY_MIGRATION_FUNCTIONS = { + '_normalize_source_identity_label', + '_select_source_row_identity', + '_prepare_tabular_source_rows', + '_migrate_legacy_tabular_export_run', +} +FAILURE_FUNCTIONS = { + '_build_failed_tabular_generated_output_metadata', + '_build_tabular_generated_output_system_message', + '_has_generated_tabular_csv_output', + '_normalize_generated_analysis_artifact_metadata', +} +ARTIFACT_FUNCTIONS = {'_upload_generated_chat_artifact_for_current_user'} +SCHEDULER_FUNCTIONS = {'_query_scheduler_candidates_by_status'} + + +def _load_contract_helpers(): + """Load the pure row-contract helpers without importing the Flask app.""" + source_text = EXPORT_MODULE.read_text(encoding='utf-8') + module_tree = ast.parse(source_text, filename=str(EXPORT_MODULE)) + selected_nodes = [] + found_functions = set() + found_constants = set() + + for node in module_tree.body: + if isinstance(node, ast.FunctionDef) and node.name in CONTRACT_FUNCTIONS: + selected_nodes.append(node) + found_functions.add(node.name) + elif isinstance(node, ast.Assign): + assigned_names = { + target.id + for target in node.targets + if isinstance(target, ast.Name) + } + if assigned_names & CONTRACT_CONSTANTS: + selected_nodes.append(node) + found_constants.update(assigned_names & CONTRACT_CONSTANTS) + + missing_functions = CONTRACT_FUNCTIONS - found_functions + missing_constants = CONTRACT_CONSTANTS - found_constants + if missing_functions or missing_constants: + raise AssertionError( + f'Missing row-contract implementation: functions={sorted(missing_functions)}, ' + f'constants={sorted(missing_constants)}' + ) + + namespace = {'re': re, 'uuid': uuid} + extracted_module = ast.Module(body=selected_nodes, type_ignores=[]) + exec(compile(extracted_module, str(EXPORT_MODULE), 'exec'), namespace) + return namespace + + +def _get_function_node(function_name): + module_tree = ast.parse(EXPORT_MODULE.read_text(encoding='utf-8'), filename=str(EXPORT_MODULE)) + for node in module_tree.body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == function_name: + return node + raise AssertionError(f'Missing function {function_name}') + + +def _called_function_names(function_node): + return { + call.func.id + for call in ast.walk(function_node) + if isinstance(call, ast.Call) and isinstance(call.func, ast.Name) + } + + +def _load_candidate_helpers(): + """Load candidate-selection helpers with minimal invocation adapters.""" + module_tree = ast.parse(CHAT_ROUTE.read_text(encoding='utf-8'), filename=str(CHAT_ROUTE)) + selected_nodes = [ + node + for node in module_tree.body + if isinstance(node, ast.FunctionDef) and node.name in CANDIDATE_FUNCTIONS + ] + found_functions = {node.name for node in selected_nodes} + missing_functions = CANDIDATE_FUNCTIONS - found_functions + if missing_functions: + raise AssertionError(f'Missing candidate helpers: {sorted(missing_functions)}') + + def get_result_payload(invocation): + return invocation.result if isinstance(invocation.result, dict) else None + + namespace = { + 'json': json, + '_safe_int': lambda value: int(value or 0), + 'get_tabular_invocation_error_message': lambda invocation: invocation.error_message, + 'get_tabular_invocation_result_payload': get_result_payload, + } + extracted_module = ast.Module(body=selected_nodes, type_ignores=[]) + exec(compile(extracted_module, str(CHAT_ROUTE), 'exec'), namespace) + return namespace + + +def _build_query_invocation(start_row, row_count, total_matches=300, source_etag='etag-source-7'): + class InvocationPayload(dict): + pass + + rows = [ + {'Case ID': f'SC-{2001 + row_index}'} + for row_index in range(start_row, start_row + row_count) + ] + result_payload = InvocationPayload({ + 'filename': 'simplechat_row_orchestration_dataset_300.csv', + 'query_expression': 'index == index', + 'start_row': start_row, + 'returned_rows': row_count, + 'total_matches': total_matches, + 'has_more': start_row + row_count < total_matches, + 'next_start_row': start_row + row_count if start_row + row_count < total_matches else None, + 'data': rows, + }) + result_payload.internal_metadata = { + 'tabular_generated_export_source': { + 'version': 1, + 'kind': 'query_tabular_data', + 'source': 'chat', + 'container': 'personal-chat', + 'blob_path': 'user-1/conversation-1/nested/version-7/source.csv', + 'blob_etag': source_etag, + 'filename': 'simplechat_row_orchestration_dataset_300.csv', + 'query_expression': 'index == index', + 'expected_row_count': total_matches, + }, + 'tabular_source_authorization': { + 'source': 'chat', + 'scope_id': None, + 'container': 'personal-chat', + 'blob_path': 'user-1/conversation-1/nested/version-7/source.csv', + 'blob_etag': source_etag, + }, + } + return SimpleNamespace( + plugin_name='TabularProcessingPlugin', + function_name='query_tabular_data', + parameters={ + 'filename': 'simplechat_row_orchestration_dataset_300.csv', + 'query_expression': 'index == index', + 'source': 'chat', + 'start_row': str(start_row), + 'max_rows': '100', + }, + result=result_payload, + error_message=None, + ) + + +class _CountingTextSink: + """Text sink that records write bounds without retaining generated output.""" + + def __init__(self): + self.write_count = 0 + self.total_char_count = 0 + self.max_write_chars = 0 + + def write(self, value): + value_length = len(value) + self.write_count += 1 + self.total_char_count += value_length + self.max_write_chars = max(self.max_write_chars, value_length) + return value_length + + +def _load_stream_writer(download_json_blob): + module_tree = ast.parse(EXPORT_MODULE.read_text(encoding='utf-8'), filename=str(EXPORT_MODULE)) + selected_nodes = [ + node + for node in module_tree.body + if isinstance(node, ast.FunctionDef) and node.name in STREAM_FUNCTIONS + ] + if len(selected_nodes) != len(STREAM_FUNCTIONS): + raise AssertionError('Missing bounded output stream writer') + + namespace = { + 'csv': csv, + 'json': json, + '_safe_int': lambda value: int(value or 0), + '_output_blob_path': lambda user_id, conversation_id, run_id, batch_number: batch_number, + '_download_json_blob': download_json_blob, + 'TABULAR_EXPORT_OUTPUT_ROW_NUMBER_FIELD': 'source_row_number', + } + extracted_module = ast.Module(body=selected_nodes, type_ignores=[]) + exec(compile(extracted_module, str(EXPORT_MODULE), 'exec'), namespace) + return namespace['_write_ordered_output_stream'] + + +class _SourceReaderPlugin: + """Small plugin stand-in that records the largest source chunk received.""" + + max_chunk_rows = 0 + + def _normalize_dataframe_columns(self, dataframe): + self.__class__.max_chunk_rows = max(self.__class__.max_chunk_rows, len(dataframe)) + normalized = dataframe.copy() + normalized.columns = [str(column).strip() for column in normalized.columns] + return normalized + + def _apply_query_expression_with_fallback(self, dataframe, query_expression=None, normalize_match=False): + del normalize_match + return dataframe.query(query_expression) if query_expression else dataframe, False + + def _parse_optional_column_list_argument(self, columns): + if not columns: + return None + return [column.strip() for column in str(columns).split(',') if column.strip()] + + def _build_row_output_records(self, dataframe, selected_columns): + return dataframe[selected_columns].to_dict(orient='records') + + +def _load_source_reader_helpers(): + module_spec = importlib.util.spec_from_file_location('functions_tabular_csv_query_test', CSV_QUERY_MODULE) + query_module = importlib.util.module_from_spec(module_spec) + module_spec.loader.exec_module(query_module) + return { + function_name: getattr(query_module, function_name) + for function_name in SOURCE_READER_FUNCTIONS + } | {'module': query_module} + + +def _load_authorization_helper(conversation_owner, visible_public_ids=None, group_authorizer=None): + class CosmosResourceNotFoundError(Exception): + pass + + class ConversationContainer: + def read_item(self, item, partition_key): + assert item == partition_key + return {'id': item, 'user_id': conversation_owner} + + module_tree = ast.parse(EXPORT_MODULE.read_text(encoding='utf-8'), filename=str(EXPORT_MODULE)) + selected_nodes = [ + node + for node in module_tree.body + if isinstance(node, ast.FunctionDef) and node.name in AUTHORIZATION_FUNCTIONS + ] + if len(selected_nodes) != len(AUTHORIZATION_FUNCTIONS): + raise AssertionError('Missing export execution authorization helper') + + namespace = { + 'cosmos_conversations_container': ConversationContainer(), + 'CosmosResourceNotFoundError': CosmosResourceNotFoundError, + 'storage_account_personal_chat_container_name': 'personal-chat', + 'storage_account_user_documents_container_name': 'user-documents', + 'storage_account_group_documents_container_name': 'group-documents', + 'storage_account_public_documents_container_name': 'public-documents', + 'assert_group_role': group_authorizer or (lambda *args, **kwargs: 'User'), + 'get_user_visible_public_workspace_ids_from_settings': lambda user_id: list(visible_public_ids or []), + } + extracted_module = ast.Module(body=selected_nodes, type_ignores=[]) + exec(compile(extracted_module, str(EXPORT_MODULE), 'exec'), namespace) + return namespace['_authorize_tabular_export_run_execution'] + + +def _load_cancellation_helpers(initial_run): + class CosmosResourceNotFoundError(Exception): + pass + + stored_run = dict(initial_run) + module_tree = ast.parse(EXPORT_MODULE.read_text(encoding='utf-8'), filename=str(EXPORT_MODULE)) + selected_nodes = [ + node + for node in module_tree.body + if isinstance(node, ast.FunctionDef) and node.name in CANCELLATION_FUNCTIONS + ] + if len(selected_nodes) != len(CANCELLATION_FUNCTIONS): + raise AssertionError('Missing durable export cancellation helpers') + + def read_run(user_id, run_id): + assert user_id == stored_run['user_id'] + assert run_id == stored_run['id'] + return dict(stored_run) + + def replace_run(run): + stored_run.clear() + stored_run.update(run) + return dict(stored_run) + + namespace = { + 'logging': logging, + 'CosmosResourceNotFoundError': CosmosResourceNotFoundError, + 'TABULAR_EXPORT_STATUS_COMPLETED': 'completed', + 'TABULAR_EXPORT_STATUS_CANCELED': 'canceled', + 'get_settings': lambda: {}, + '_read_run': read_run, + '_replace_run': replace_run, + '_now_iso': lambda: '2026-07-21T18:00:00+00:00', + 'log_event': lambda *args, **kwargs: None, + } + extracted_module = ast.Module(body=selected_nodes, type_ignores=[]) + exec(compile(extracted_module, str(EXPORT_MODULE), 'exec'), namespace) + namespace['_build_run_public_status'] = lambda run, settings=None: { + 'status': run.get('status'), + 'can_cancel': namespace['_can_cancel_run'](run), + } + return namespace, stored_run + + +def _load_summary_helpers(batch_summaries): + module_tree = ast.parse(EXPORT_MODULE.read_text(encoding='utf-8'), filename=str(EXPORT_MODULE)) + selected_nodes = [ + node + for node in module_tree.body + if isinstance(node, ast.FunctionDef) and node.name in SUMMARY_FUNCTIONS + ] + if len(selected_nodes) != len(SUMMARY_FUNCTIONS): + raise AssertionError('Missing bounded post-run summary helpers') + + namespace = { + 'Counter': Counter, + '_safe_int': lambda value: int(value or 0), + 'TABULAR_EXPORT_OUTPUT_ROW_NUMBER_FIELD': 'source_row_number', + 'TABULAR_EXPORT_OUTPUT_ROW_IDENTITY_FIELD': 'source_row_identity', + 'TABULAR_EXPORT_SUMMARY_MAX_FIELDS': 25, + 'TABULAR_EXPORT_SUMMARY_MAX_VALUES_PER_FIELD': 5, + 'TABULAR_EXPORT_SUMMARY_AGGREGATE_MAX_VALUES': 25, + '_output_summary_blob_path': lambda user_id, conversation_id, run_id, batch_number: batch_number, + '_blob_exists': lambda batch_number: batch_number in batch_summaries, + '_download_json_blob': lambda batch_number: batch_summaries[batch_number], + } + extracted_module = ast.Module(body=selected_nodes, type_ignores=[]) + exec(compile(extracted_module, str(EXPORT_MODULE), 'exec'), namespace) + return namespace + + +def _load_fencing_helpers(current_run, replace_error=None): + class TabularExportCanceledError(RuntimeError): + pass + + class TabularExportLeaseLostError(RuntimeError): + pass + + module_tree = ast.parse(EXPORT_MODULE.read_text(encoding='utf-8'), filename=str(EXPORT_MODULE)) + selected_nodes = [ + node + for node in module_tree.body + if isinstance(node, ast.FunctionDef) and node.name in FENCING_FUNCTIONS + ] + if len(selected_nodes) != len(FENCING_FUNCTIONS): + raise AssertionError('Missing lease fencing helpers') + + namespace = { + 'TABULAR_EXPORT_STATUS_CANCELED': 'canceled', + 'TABULAR_EXPORT_STATUS_RUNNING': 'running', + 'TabularExportCanceledError': TabularExportCanceledError, + 'TabularExportLeaseLostError': TabularExportLeaseLostError, + '_safe_int': lambda value: int(value or 0), + '_read_run': lambda user_id, run_id: dict(current_run), + '_replace_run': lambda run: (_ for _ in ()).throw(replace_error) if replace_error else dict(run), + } + extracted_module = ast.Module(body=selected_nodes, type_ignores=[]) + exec(compile(extracted_module, str(EXPORT_MODULE), 'exec'), namespace) + return namespace + + +def _load_legacy_migration_helper(aggregate_batches): + module_tree = ast.parse(EXPORT_MODULE.read_text(encoding='utf-8'), filename=str(EXPORT_MODULE)) + selected_nodes = [ + node + for node in module_tree.body + if isinstance(node, ast.FunctionDef) and node.name in LEGACY_MIGRATION_FUNCTIONS + ] + if len(selected_nodes) != len(LEGACY_MIGRATION_FUNCTIONS): + raise AssertionError('Missing legacy export migration helpers') + + uploaded_batches = {} + deleted_blobs = [] + namespace = { + 're': re, + 'uuid': uuid, + 'TABULAR_EXPORT_CONTRACT_VERSION': 2, + 'TABULAR_EXPORT_INPUT_ROW_NUMBER_FIELD': '__simplechat_source_row_number', + 'TABULAR_EXPORT_INPUT_ROW_IDENTITY_FIELD': '__simplechat_source_row_identity', + 'TABULAR_EXPORT_INPUT_ROW_TOKEN_FIELD': '__simplechat_source_row_token', + '_safe_int': lambda value: int(value or 0), + '_download_json_blob': lambda path: aggregate_batches if path == 'legacy-input.json' else uploaded_batches[path], + '_upload_json_blob': lambda path, payload, metadata=None: uploaded_batches.__setitem__(path, payload), + '_input_blob_path': lambda user_id, conversation_id, run_id, batch_number: f'batch-{batch_number}', + '_output_blob_path': lambda user_id, conversation_id, run_id, batch_number: f'output-{batch_number}', + '_output_summary_blob_path': lambda user_id, conversation_id, run_id, batch_number: f'summary-{batch_number}', + '_delete_blob_if_exists': deleted_blobs.append, + '_now_iso': lambda: '2026-07-21T18:00:00+00:00', + '_raise_if_tabular_export_canceled': lambda run: run, + '_replace_claimed_run': lambda run: dict(run), + } + extracted_module = ast.Module(body=selected_nodes, type_ignores=[]) + exec(compile(extracted_module, str(EXPORT_MODULE), 'exec'), namespace) + return namespace['_migrate_legacy_tabular_export_run'], uploaded_batches, deleted_blobs + + +def _load_failed_export_helpers(): + module_tree = ast.parse(CHAT_ROUTE.read_text(encoding='utf-8'), filename=str(CHAT_ROUTE)) + selected_nodes = [ + node + for node in module_tree.body + if isinstance(node, ast.FunctionDef) and node.name in FAILURE_FUNCTIONS + ] + if len(selected_nodes) != len(FAILURE_FUNCTIONS): + raise AssertionError('Missing explicit failed-export fallback helpers') + + namespace = { + '_safe_int': lambda value: int(value or 0), + '_build_tabular_generated_output_file_name': ( + lambda filename, output_format: f"{Path(filename).stem}_generated.{output_format}" + ), + } + extracted_module = ast.Module(body=selected_nodes, type_ignores=[]) + exec(compile(extracted_module, str(CHAT_ROUTE), 'exec'), namespace) + return namespace + + +def _load_idempotent_artifact_helper(): + class CosmosResourceNotFoundError(Exception): + pass + + class ConversationContainer: + def read_item(self, item, partition_key): + return {'id': item, 'user_id': 'user-1'} + + class MessageContainer: + def __init__(self): + self.items = {} + self.upsert_count = 0 + + def read_item(self, item, partition_key): + if item not in self.items: + raise CosmosResourceNotFoundError() + return dict(self.items[item]) + + def upsert_item(self, item): + self.items[item['id']] = dict(item) + self.upsert_count += 1 + return dict(item) + + class BlobClient: + def __init__(self): + self.uploaded = False + + def exists(self): + return self.uploaded + + def upload_blob(self, content, overwrite, metadata): + del content, overwrite, metadata + self.uploaded = True + + class BlobServiceClient: + def __init__(self): + self.clients = {} + + def get_blob_client(self, container, blob): + return self.clients.setdefault((container, blob), BlobClient()) + + module_tree = ast.parse(SIMPLECHAT_OPERATIONS.read_text(encoding='utf-8'), filename=str(SIMPLECHAT_OPERATIONS)) + selected_nodes = [ + node + for node in module_tree.body + if isinstance(node, ast.FunctionDef) and node.name in ARTIFACT_FUNCTIONS + ] + if len(selected_nodes) != len(ARTIFACT_FUNCTIONS): + raise AssertionError('Missing idempotent artifact helper') + + message_container = MessageContainer() + blob_service_client = BlobServiceClient() + namespace = { + 'Any': Any, + 'Dict': Dict, + 'Optional': Optional, + 'CLIENTS': {'storage_account_office_docs_client': blob_service_client}, + 'CosmosResourceNotFoundError': CosmosResourceNotFoundError, + 'TABULAR_EXTENSIONS': {'csv'}, + 'cosmos_conversations_container': ConversationContainer(), + 'cosmos_messages_container': message_container, + 'datetime': datetime, + 'timezone': timezone, + 'os': os, + 'uuid': uuid, + 'storage_account_personal_chat_container_name': 'personal-chat', + '_get_latest_personal_thread_id': lambda conversation_id: None, + 'log_event': lambda *args, **kwargs: None, + } + extracted_module = ast.Module(body=selected_nodes, type_ignores=[]) + exec(compile(extracted_module, str(SIMPLECHAT_OPERATIONS), 'exec'), namespace) + return namespace['_upload_generated_chat_artifact_for_current_user'], message_container + + +def _load_scheduler_query_helper(runs): + class RunContainer: + def __init__(self): + self.query = None + + def query_items(self, query, parameters, enable_cross_partition_query): + del parameters, enable_cross_partition_query + self.query = query + return iter(runs) + + module_tree = ast.parse(EXPORT_MODULE.read_text(encoding='utf-8'), filename=str(EXPORT_MODULE)) + selected_nodes = [ + node + for node in module_tree.body + if isinstance(node, ast.FunctionDef) and node.name in SCHEDULER_FUNCTIONS + ] + if len(selected_nodes) != len(SCHEDULER_FUNCTIONS): + raise AssertionError('Missing scheduler candidate query helper') + + run_container = RunContainer() + namespace = { + 'TABULAR_EXPORT_DEFAULT_SCAN_LIMIT': 5, + 'TABULAR_EXPORT_RUN_TYPE': 'tabular_generated_output_run', + '_safe_int': lambda value, default=0, minimum=None, maximum=None: max( + minimum or int(value or default), + min(maximum or int(value or default), int(value or default)), + ), + '_scheduler_candidate_reason': lambda run, settings: 'eligible' if run.get('eligible') else None, + 'cosmos_tabular_export_runs_container': run_container, + 'log_event': lambda *args, **kwargs: None, + 'logging': logging, + } + extracted_module = ast.Module(body=selected_nodes, type_ignores=[]) + exec(compile(extracted_module, str(EXPORT_MODULE), 'exec'), namespace) + return namespace['_query_scheduler_candidates_by_status'], run_container + + +def test_source_identity_and_order_contract(): + """Every row receives a canonical ordinal and preserves its source identifier.""" + helpers = _load_contract_helpers() + source_rows = [ + {'Case ID': f'SC-{2001 + index}', 'Comment': f'row {index + 1}'} + for index in range(300) + ] + + prepared_rows = helpers['_prepare_tabular_source_rows'](source_rows, start_row=0) + + assert len(prepared_rows) == 300 + assert prepared_rows[0]['__simplechat_source_row_number'] == 1 + assert prepared_rows[-1]['__simplechat_source_row_number'] == 300 + assert prepared_rows[0]['__simplechat_source_row_identity'] == 'SC-2001' + assert prepared_rows[-1]['__simplechat_source_row_identity'] == 'SC-2300' + assert len({row['__simplechat_source_row_token'] for row in prepared_rows}) == 300 + assert [row['__simplechat_source_row_number'] for row in prepared_rows] == list(range(1, 301)) + + +def test_generated_batch_schema_contract(): + """Batch output is ordered by the first schema and source identity is authoritative.""" + helpers = _load_contract_helpers() + prepared_rows = helpers['_prepare_tabular_source_rows']( + [ + {'Case ID': 'SC-2001', 'Comment': 'first'}, + {'Case ID': 'SC-2002', 'Comment': 'second'}, + ], + start_row=0, + ) + generated_entries = [ + { + '__simplechat_source_row_token': prepared_rows[0]['__simplechat_source_row_token'], + 'answer': 'yes', + 'risk': 'low', + }, + { + '__simplechat_source_row_token': prepared_rows[1]['__simplechat_source_row_token'], + 'risk': 'high', + 'answer': 'no', + }, + ] + + normalized_entries, output_schema = helpers['_normalize_generated_batch_entries']( + prepared_rows, + generated_entries, + ) + + assert output_schema == ['source_row_number', 'source_row_identity', 'answer', 'risk'] + assert list(normalized_entries[1]) == output_schema + assert normalized_entries[0]['source_row_number'] == 1 + assert normalized_entries[1]['source_row_identity'] == 'SC-2002' + + try: + helpers['_normalize_generated_batch_entries']( + prepared_rows, + [ + { + '__simplechat_source_row_token': prepared_rows[0]['__simplechat_source_row_token'], + 'answer': 'yes', + 'risk': 'low', + }, + { + '__simplechat_source_row_token': prepared_rows[1]['__simplechat_source_row_token'], + 'answer': 'no', + 'unexpected': 'schema drift', + }, + ], + expected_output_schema=output_schema, + ) + except ValueError as exc: + assert 'schema' in str(exc).lower() + else: + raise AssertionError('Schema drift must fail before a batch is checkpointed') + + try: + helpers['_normalize_generated_batch_entries']( + prepared_rows, + [ + { + '__simplechat_source_row_token': prepared_rows[1]['__simplechat_source_row_token'], + 'answer': 'no', + 'risk': 'high', + }, + { + '__simplechat_source_row_token': prepared_rows[0]['__simplechat_source_row_token'], + 'answer': 'yes', + 'risk': 'low', + }, + ], + expected_output_schema=output_schema, + ) + except ValueError as exc: + assert 'token mismatch' in str(exc).lower() + else: + raise AssertionError('Swapped model rows must fail source-token validation') + + +def test_durable_runner_enforces_row_contract(): + """Queueing, generation, and checkpointing must all enforce the shared contract.""" + queue_calls = _called_function_names(_get_function_node('queue_tabular_generated_output_run')) + generation_calls = _called_function_names(_get_function_node('_generate_batch_entries')) + checkpoint_source = ast.unparse(_get_function_node('_checkpoint_generated_batch_results')) + process_source = ast.unparse(_get_function_node('process_tabular_generated_output_run')) + + assert '_prepare_tabular_source_rows' in queue_calls + assert '_normalize_generated_batch_entries' in generation_calls + assert "run['output_schema']" in checkpoint_source + assert "run.get('output_schema')" in process_source + assert 'window_end = window_start' in process_source + assert process_source.index('_authorize_tabular_export_run_execution') < process_source.index( + '_migrate_legacy_tabular_export_run' + ) + resume_source = ast.unparse(_get_function_node('resume_tabular_generated_output_run')) + assert resume_source.index('_authorize_tabular_export_run_execution') < resume_source.index('run.update') + complete_source = ast.unparse(_get_function_node('_complete_run')) + assert complete_source.index('_write_ordered_output_stream') < complete_source.index( + 'upload_generated_analysis_artifact_stream_for_user' + ) + assert complete_source.index('upload_generated_analysis_artifact_stream_for_user') < complete_source.index( + "'status': TABULAR_EXPORT_STATUS_COMPLETED" + ) + + +def test_paginated_candidate_coalesces_all_300_rows(): + """Compatible tool pages form one exhaustive, ordered export source.""" + helpers = _load_candidate_helpers() + invocations = [ + _build_query_invocation(0, 94), + _build_query_invocation(94, 95), + _build_query_invocation(189, 94), + _build_query_invocation(283, 17), + ] + + candidate = helpers['_build_tabular_generated_output_source_candidate'](invocations) + + assert candidate['full_result_available'] is True + assert candidate['row_count'] == 300 + assert len(candidate['rows']) == 300 + assert candidate['rows'][0]['Case ID'] == 'SC-2001' + assert candidate['rows'][-1]['Case ID'] == 'SC-2300' + assert candidate['page_count'] == 4 + assert candidate['source_descriptor']['blob_path'] == ( + 'user-1/conversation-1/nested/version-7/source.csv' + ) + assert candidate['source_descriptor']['blob_etag'] == 'etag-source-7' + assert candidate['source_authorization']['container'] == 'personal-chat' + + +def test_paginated_candidate_rejects_gaps(): + """Missing source intervals remain incomplete instead of producing a partial export.""" + helpers = _load_candidate_helpers() + invocations = [ + _build_query_invocation(0, 94), + _build_query_invocation(95, 205), + ] + + candidate = helpers['_build_tabular_generated_output_source_candidate'](invocations) + + assert candidate['full_result_available'] is False + assert candidate['validation_error'] + assert 'gap' in candidate['validation_error'].lower() + + +def test_paginated_candidate_rejects_mixed_source_versions(): + """Contiguous pages from different blob ETags cannot form an exhaustive source.""" + helpers = _load_candidate_helpers() + invocations = [ + _build_query_invocation(0, 150, source_etag='etag-source-7'), + _build_query_invocation(150, 150, source_etag='etag-source-8'), + ] + + candidate = helpers['_build_tabular_generated_output_source_candidate'](invocations) + + assert candidate['full_result_available'] is False + assert 'different source blobs or versions' in candidate['validation_error'].lower() + + +def test_streaming_finalizer_writes_30000_rows_in_bounded_chunks(): + """Final assembly reads one checkpoint at a time and never builds a full row list.""" + requested_batches = [] + + def download_batch(batch_number): + requested_batches.append(batch_number) + first_row_number = ((batch_number - 1) * 50) + 1 + return [ + { + 'source_row_number': first_row_number + offset, + 'source_row_identity': f'SC-{first_row_number + offset}', + 'answer': 'yes', + } + for offset in range(50) + ] + + write_output = _load_stream_writer(download_batch) + output_sink = _CountingTextSink() + run = { + 'id': 'run-30000', + 'user_id': 'user-1', + 'conversation_id': 'conversation-1', + 'output_format': 'csv', + 'row_count': 30000, + 'batch_count': 600, + 'output_schema': ['source_row_number', 'source_row_identity', 'answer'], + } + + written_rows = write_output(run, output_sink) + + assert written_rows == 30000 + assert requested_batches == list(range(1, 601)) + assert output_sink.total_char_count > 30000 + assert output_sink.max_write_chars < 1000 + + +def test_streaming_finalizer_rejects_source_order_gaps(): + """An ordinal gap fails final validation before the artifact is published.""" + def download_batch(batch_number): + if batch_number == 1: + return [ + {'source_row_number': 1, 'source_row_identity': 'SC-1', 'answer': 'yes'}, + {'source_row_number': 2, 'source_row_identity': 'SC-2', 'answer': 'yes'}, + ] + return [ + {'source_row_number': 4, 'source_row_identity': 'SC-4', 'answer': 'yes'}, + ] + + write_output = _load_stream_writer(download_batch) + run = { + 'id': 'run-gap', + 'user_id': 'user-1', + 'conversation_id': 'conversation-1', + 'output_format': 'json', + 'row_count': 3, + 'batch_count': 2, + 'output_schema': ['source_row_number', 'source_row_identity', 'answer'], + } + + try: + write_output(run, _CountingTextSink()) + except ValueError as exc: + assert 'order' in str(exc).lower() or 'gap' in str(exc).lower() + else: + raise AssertionError('Finalization must reject a source ordinal gap') + + +def test_csv_query_source_reader_scales_and_resumes(): + """CSV replay scans bounded chunks and can resume from a physical source row.""" + helpers = _load_source_reader_helpers() + _SourceReaderPlugin.max_chunk_rows = 0 + csv_content = 'Case ID,Score\n' + ''.join( + f'SC-{row_number},{row_number}\n' + for row_number in range(1, 30001) + ) + descriptor = { + 'query_expression': 'Score >= 0', + 'return_columns': 'Case ID,Score', + } + + original_read_csv = helpers['module'].pandas.read_csv + observed_skiprows = [] + + def recording_read_csv(*args, **kwargs): + observed_skiprows.append(kwargs.get('skiprows')) + return original_read_csv(*args, **kwargs) + + helpers['module'].pandas.read_csv = recording_read_csv + try: + row_iterator = helpers['iter_tabular_csv_query_rows']( + csv_stream=io.StringIO(csv_content), + query_expression=descriptor['query_expression'], + return_columns=descriptor['return_columns'], + source_chunk_rows=257, + tabular_plugin=_SourceReaderPlugin(), + start_source_row=15000, + ) + row_count = 0 + first_result = None + last_result = None + for result in row_iterator: + row_count += 1 + first_result = first_result or result + last_result = result + finally: + helpers['module'].pandas.read_csv = original_read_csv + + assert row_count == 15000 + assert first_result == (15001, {'Case ID': 'SC-15001', 'Score': 15001}) + assert last_result == (30000, {'Case ID': 'SC-30000', 'Score': 30000}) + assert _SourceReaderPlugin.max_chunk_rows <= 257 + assert any(callable(skiprows) for skiprows in observed_skiprows) + + assert helpers['validate_tabular_csv_query_expression']( + '`Case ID` in ["SC-1", "SC-2"] and Score >= 0' + ) + try: + helpers['validate_tabular_csv_query_expression']('Score > Score.mean()') + except ValueError as exc: + assert 'bounded chunks' in str(exc) + else: + raise AssertionError('Cross-row aggregation calls must be rejected') + + try: + list(helpers['iter_tabular_csv_query_rows']( + csv_stream=io.StringIO(csv_content), + query_expression='Score > @dynamic_threshold', + return_columns=None, + source_chunk_rows=257, + tabular_plugin=_SourceReaderPlugin(), + )) + except ValueError as exc: + assert 'bounded chunks' in str(exc) + else: + raise AssertionError('Cross-chunk query context must be rejected for durable replay') + + +def test_worker_revalidates_conversation_and_workspace_authorization(): + """Stored source descriptors are authorized again whenever a worker executes.""" + authorize_personal = _load_authorization_helper('user-1') + personal_run = { + 'id': 'run-personal', + 'user_id': 'user-1', + 'conversation_id': 'conversation-1', + 'source_descriptor': { + 'source': 'chat', + 'container': 'personal-chat', + 'blob_path': 'user-1/conversation-1/source.csv', + }, + } + assert authorize_personal(personal_run)['user_id'] == 'user-1' + + authorize_wrong_owner = _load_authorization_helper('different-user') + try: + authorize_wrong_owner(personal_run) + except PermissionError as exc: + assert 'ownership' in str(exc).lower() + else: + raise AssertionError('A worker must reject a conversation whose ownership changed') + + authorized_groups = [] + + def authorize_group(user_id, group_id, allowed_roles): + authorized_groups.append((user_id, group_id, tuple(allowed_roles))) + return 'User' + + authorize_group_run = _load_authorization_helper('user-1', group_authorizer=authorize_group) + group_run = { + 'id': 'run-group', + 'user_id': 'user-1', + 'conversation_id': 'conversation-1', + 'source_descriptor': { + 'source': 'group', + 'scope_id': 'group-1', + 'container': 'group-documents', + 'blob_path': 'group-1/source.csv', + }, + } + authorize_group_run(group_run) + assert authorized_groups == [ + ('user-1', 'group-1', ('Owner', 'Admin', 'DocumentManager', 'User')), + ] + + authorize_public = _load_authorization_helper('user-1', visible_public_ids=['public-1']) + public_run = { + 'id': 'run-public', + 'user_id': 'user-1', + 'conversation_id': 'conversation-1', + 'source_descriptor': { + 'source': 'public', + 'scope_id': 'public-1', + 'container': 'public-documents', + 'blob_path': 'public-1/source.csv', + }, + } + authorize_public(public_run) + + +def test_durable_cancellation_is_idempotent_and_terminal(): + """Cancel persists a terminal state that cannot be resumed or canceled again.""" + helpers, stored_run = _load_cancellation_helpers({ + 'id': 'run-cancel', + 'user_id': 'user-1', + 'conversation_id': 'conversation-1', + 'status': 'running', + 'completed_batches': 25, + 'processed_rows': 1250, + }) + + cancel_result = helpers['cancel_tabular_generated_output_run']('user-1', 'run-cancel') + assert cancel_result['success'] is True + assert cancel_result['canceled'] is True + assert cancel_result['run']['can_cancel'] is False + assert stored_run['status'] == 'canceled' + assert stored_run['next_attempt_at'] is None + + repeated_result = helpers['cancel_tabular_generated_output_run']('user-1', 'run-cancel') + assert repeated_result['success'] is True + assert repeated_result['message'] == 'Background export is already canceled.' + + +def test_worker_lease_fencing_rejects_stale_claims(): + """A reclaimed or canceled run stops the stale worker before it mutates state.""" + owned_run = { + 'id': 'run-fenced', + 'user_id': 'user-1', + 'status': 'running', + 'lease_holder_id': 'worker-a', + 'lease_generation': 3, + '_etag': 'etag-3', + } + helpers = _load_fencing_helpers(owned_run) + local_run = dict(owned_run) + helpers['_raise_if_tabular_export_canceled'](local_run) + assert local_run['_etag'] == 'etag-3' + + reclaimed_run = dict(owned_run, lease_holder_id='worker-b', lease_generation=4, _etag='etag-4') + helpers = _load_fencing_helpers(reclaimed_run) + try: + helpers['_raise_if_tabular_export_canceled'](dict(owned_run)) + except RuntimeError as exc: + assert 'lost its claim' in str(exc) + else: + raise AssertionError('A stale worker must lose its lease fence') + + canceled_run = dict(owned_run, status='canceled', _etag='etag-canceled') + helpers = _load_fencing_helpers(canceled_run) + try: + helpers['_raise_if_tabular_export_canceled'](dict(owned_run)) + except RuntimeError as exc: + assert 'canceled' in str(exc) + else: + raise AssertionError('A canceled run must stop its worker') + + class PreconditionFailed(Exception): + status_code = 412 + + helpers = _load_fencing_helpers(owned_run, replace_error=PreconditionFailed()) + try: + helpers['_replace_claimed_run'](dict(owned_run)) + except RuntimeError as exc: + assert 'lost its claim' in str(exc) + else: + raise AssertionError('An ETag conflict must fence the stale worker') + + +def test_legacy_run_migration_tokenizes_inputs_and_resets_outputs(): + """Pre-contract runs migrate deterministic inputs and regenerate old outputs.""" + aggregate_batches = [ + [{'Case ID': 'SC-1'}, {'Case ID': 'SC-2'}], + [{'Case ID': 'SC-3'}], + ] + migrate_run, uploaded_batches, deleted_blobs = _load_legacy_migration_helper(aggregate_batches) + migrated_run = migrate_run({ + 'id': 'legacy-run', + 'user_id': 'user-1', + 'conversation_id': 'conversation-1', + 'row_count': 3, + 'batch_count': 2, + 'completed_batches': 1, + 'processed_rows': 2, + 'input_blob_path': 'legacy-input.json', + }) + + assert migrated_run['contract_version'] == 2 + assert migrated_run['completed_batches'] == 0 + assert migrated_run['processed_rows'] == 0 + assert migrated_run['output_schema'] is None + assert migrated_run['regenerate_legacy_output_checkpoints'] is False + assert migrated_run['input_blob_path'] is None + assert [row['__simplechat_source_row_number'] for row in uploaded_batches['batch-1']] == [1, 2] + assert uploaded_batches['batch-2'][0]['__simplechat_source_row_number'] == 3 + assert deleted_blobs == ['output-1', 'summary-1', 'output-2', 'summary-2'] + migrate_again, second_uploaded_batches, _ = _load_legacy_migration_helper(aggregate_batches) + migrate_again({ + 'id': 'legacy-run', + 'user_id': 'user-1', + 'conversation_id': 'conversation-1', + 'row_count': 3, + 'batch_count': 2, + 'completed_batches': 1, + 'processed_rows': 2, + 'input_blob_path': 'legacy-input.json', + }) + assert uploaded_batches['batch-1'][0]['__simplechat_source_row_token'] == ( + second_uploaded_batches['batch-1'][0]['__simplechat_source_row_token'] + ) + + +def test_post_run_summary_uses_only_bounded_batch_summaries(): + """Thirty-thousand-row overall analysis merges compact checkpoint summaries only.""" + batch_summaries = {} + helpers = _load_summary_helpers(batch_summaries) + for batch_number in range(1, 601): + first_row_number = ((batch_number - 1) * 50) + 1 + batch_entries = [ + { + 'source_row_number': first_row_number + offset, + 'source_row_identity': f'SC-{first_row_number + offset}', + 'answer': 'yes' if offset % 2 == 0 else 'no', + 'risk': 'low' if offset < 40 else 'high', + } + for offset in range(50) + ] + batch_summaries[batch_number] = helpers['_build_generated_batch_summary'](batch_entries) + + summary = helpers['_build_compact_post_run_summary']({ + 'id': 'run-summary', + 'user_id': 'user-1', + 'conversation_id': 'conversation-1', + 'row_count': 30000, + 'batch_count': 600, + }) + + assert 'Processed 30,000 ordered row(s) across 600 checkpointed batch(es).' in summary + assert 'answer 100% populated' in summary + assert 'risk 100% populated' in summary + assert 'answer: yes (15,000), no (15,000)' in summary + assert len(summary) < 2000 + + +def test_final_artifact_publication_is_retry_idempotent(): + """Retrying final publication reuses one deterministic artifact message and blob.""" + upload_artifact, message_container = _load_idempotent_artifact_helper() + upload_arguments = { + 'current_user_id': 'user-1', + 'conversation_id': 'conversation-1', + 'normalized_file_name': 'generated.csv', + 'file_content_bytes': b'source_row_number,answer\n1,yes\n', + 'artifact_metadata': { + 'capability': 'tabular', + 'output_format': 'csv', + 'summary': 'Processed 1 ordered row.', + }, + 'artifact_idempotency_key': 'tabular-generated-output:run-1', + } + + first_result = upload_artifact(**upload_arguments) + second_result = upload_artifact(**upload_arguments) + + assert first_result['message']['id'] == second_result['message']['id'] + assert first_result['message']['blob_path'] == second_result['message']['blob_path'] + assert message_container.upsert_count == 1 + + +def test_scheduler_filters_before_limiting_candidates(): + """Ineligible rows before the scan limit cannot hide an older recoverable run.""" + runs = [ + {'id': f'healthy-{index}', 'eligible': False} + for index in range(1, 7) + ] + [ + {'id': 'stale-run', 'eligible': True}, + ] + query_candidates, run_container = _load_scheduler_query_helper(runs) + + candidates = query_candidates('running', 1, settings={}) + + assert [candidate['id'] for candidate in candidates] == ['stale-run'] + assert 'TOP' not in run_container.query.upper() + assert 'ORDER BY c.updated_at ASC' in run_container.query + + +def test_route_queues_replayable_pages_and_suppresses_summary_fallback(): + """Multi-page runs expose durable metadata that blocks assistant-table fallback exports.""" + route_source = CHAT_ROUTE.read_text(encoding='utf-8') + assert 'should_queue_source_backed_run' in route_source + assert 'should_queue_materialized_pages' in route_source + assert 'exceeds_background_threshold' in route_source + assert 'source_descriptor=source_descriptor' in route_source + assert "output_metadata.get('background_export')" in route_source + assert '_has_generated_tabular_csv_output(existing_outputs)' in route_source + + helpers = _load_failed_export_helpers() + failed_output = helpers['_build_failed_tabular_generated_output_metadata']( + { + 'filename': 'source.csv', + 'total_matches': 300, + }, + 'json', + 'Source replay failed. No partial export was created.', + ) + assert failed_output['status'] == 'failed' + assert failed_output['background_export'] is True + assert failed_output['suppress_assistant_table_export'] is True + assert helpers['_has_generated_tabular_csv_output']([failed_output]) is True + normalized_failure = helpers['_normalize_generated_analysis_artifact_metadata']( + failed_output, + default_capability='tabular', + ) + assert normalized_failure['status'] == 'failed' + assert normalized_failure['background_export'] is True + assert not normalized_failure.get('export_run_id') + failure_handoff = helpers['_build_tabular_generated_output_system_message'](failed_output) + assert 'failed' in failure_handoff.lower() + assert 'do not recreate' in failure_handoff.lower() + chat_messages_source = ( + REPO_ROOT / 'application' / 'single_app' / 'static' / 'js' / 'chat' / 'chat-messages.js' + ).read_text(encoding='utf-8') + assert 'isTerminalExportStatus' in chat_messages_source + assert 'output.suppress_assistant_table_export' in chat_messages_source + + +def main(): + """Run focused row-orchestration contract checks.""" + tests = [ + test_source_identity_and_order_contract, + test_generated_batch_schema_contract, + test_durable_runner_enforces_row_contract, + test_paginated_candidate_coalesces_all_300_rows, + test_paginated_candidate_rejects_gaps, + test_paginated_candidate_rejects_mixed_source_versions, + test_streaming_finalizer_writes_30000_rows_in_bounded_chunks, + test_streaming_finalizer_rejects_source_order_gaps, + test_csv_query_source_reader_scales_and_resumes, + test_worker_revalidates_conversation_and_workspace_authorization, + test_durable_cancellation_is_idempotent_and_terminal, + test_worker_lease_fencing_rejects_stale_claims, + test_legacy_run_migration_tokenizes_inputs_and_resets_outputs, + test_post_run_summary_uses_only_bounded_batch_summaries, + test_final_artifact_publication_is_retry_idempotent, + test_scheduler_filters_before_limiting_candidates, + test_route_queues_replayable_pages_and_suppresses_summary_fallback, + ] + for test in tests: + print(f'Running {test.__name__}...') + test() + print(f'PASS {test.__name__}') + return True + + +if __name__ == '__main__': + sys.exit(0 if main() else 1) \ No newline at end of file diff --git a/scripts/Migration-AISearch.state.json b/scripts/Migration-AISearch.state.json new file mode 100644 index 000000000..4b7311d93 --- /dev/null +++ b/scripts/Migration-AISearch.state.json @@ -0,0 +1,145 @@ +{ + "schemaVersion": 1, + "migrationType": "ai_search", + "configurationFingerprint": "e6d21189f8634b49be58598f2283f5c25609559f749175f930a1134981765ecf", + "configuration": { + "sourceService": "rbsimplechataisearch", + "sourceResourceGroup": "RG-SimpleChat", + "sourceSubscriptionId": "9698dd71-9367-49c2-bede-fd0deecfad62", + "destinationService": "migration-test-sc", + "destinationResourceGroup": "RG-Demos", + "destinationSubscriptionId": "9698dd71-9367-49c2-bede-fd0deecfad62", + "mode": "full", + "searchApiVersion": "2026-04-01", + "managementApiVersion": "2025-05-01", + "searchDnsSuffix": "search.windows.net" + }, + "status": "completed", + "createdUtc": "2026-07-21T17:03:26.0649529Z", + "updatedUtc": "2026-07-21T17:29:55.6789369Z", + "completedUtc": "2026-07-21T17:29:55.6774107Z", + "resumeCount": 0, + "currentResource": null, + "lastError": null, + "resources": { + "synonymmaps": { + "status": "completed", + "attempt": 1, + "startedUtc": "2026-07-21T17:03:26.0776967Z", + "updatedUtc": "2026-07-21T17:03:26.5299985Z", + "completedUtc": "2026-07-21T17:03:26.5299985Z", + "lastError": null, + "progress": {}, + "result": {} + }, + "index:simplechat-group-index": { + "status": "completed", + "attempt": 1, + "startedUtc": "2026-07-21T17:03:27.0215182Z", + "updatedUtc": "2026-07-21T17:25:06.2150386Z", + "completedUtc": "2026-07-21T17:25:06.2150386Z", + "lastError": null, + "progress": { + "phase": "source_documents", + "keyField": "id", + "resumeSupported": true, + "sourceDocumentCount": 53994, + "lastCommittedKey": "fff82fc7-b5a0-4665-b1b0-c9ca7af281fe_5", + "processedCount": 53994, + "copiedCount": 53994, + "skippedCount": 0, + "batchCount": 541 + }, + "result": { + "CopiedCount": 53994, + "SkippedCount": 0, + "ProcessedCount": 53994, + "TotalCount": 53994, + "BatchCount": 541 + } + }, + "index:simplechat-public-index": { + "status": "completed", + "attempt": 1, + "startedUtc": "2026-07-21T17:25:06.2242466Z", + "updatedUtc": "2026-07-21T17:25:15.5708219Z", + "completedUtc": "2026-07-21T17:25:15.5708219Z", + "lastError": null, + "progress": { + "phase": "source_documents", + "keyField": "id", + "resumeSupported": true, + "sourceDocumentCount": 321, + "lastCommittedKey": "f41a450d-119b-475d-b626-18317146d8af_1", + "processedCount": 321, + "copiedCount": 321, + "skippedCount": 0, + "batchCount": 5 + }, + "result": { + "CopiedCount": 321, + "SkippedCount": 0, + "ProcessedCount": 321, + "TotalCount": 321, + "BatchCount": 5 + } + }, + "index:simplechat-user-index": { + "status": "completed", + "attempt": 1, + "startedUtc": "2026-07-21T17:25:15.5774242Z", + "updatedUtc": "2026-07-21T17:29:53.2903790Z", + "completedUtc": "2026-07-21T17:29:53.2903790Z", + "lastError": null, + "progress": { + "phase": "source_documents", + "keyField": "id", + "resumeSupported": true, + "sourceDocumentCount": 12042, + "lastCommittedKey": "ff7e80d0-e1d5-4bae-a555-a441247e5263_1", + "processedCount": 12042, + "copiedCount": 12042, + "skippedCount": 0, + "batchCount": 122 + }, + "result": { + "CopiedCount": 12042, + "SkippedCount": 0, + "ProcessedCount": 12042, + "TotalCount": 12042, + "BatchCount": 122 + } + }, + "index:visible-rag-demo": { + "status": "completed", + "attempt": 1, + "startedUtc": "2026-07-21T17:29:53.2988790Z", + "updatedUtc": "2026-07-21T17:29:55.6680026Z", + "completedUtc": "2026-07-21T17:29:55.6680026Z", + "lastError": null, + "progress": { + "phase": "source_documents", + "keyField": "id", + "resumeSupported": true, + "sourceDocumentCount": 168, + "lastCommittedKey": "2025-sya-ll-sop-pdf-0168", + "processedCount": 168, + "copiedCount": 168, + "skippedCount": 0, + "batchCount": 3 + }, + "result": { + "CopiedCount": 168, + "SkippedCount": 0, + "ProcessedCount": 168, + "TotalCount": 168, + "BatchCount": 3 + } + } + }, + "summary": { + "IndexCount": 4, + "CopiedCount": 66525, + "SkippedCount": 0 + } +} diff --git a/scripts/Migration-Cosmos.state.json b/scripts/Migration-Cosmos.state.json new file mode 100644 index 000000000..2967ec8d4 --- /dev/null +++ b/scripts/Migration-Cosmos.state.json @@ -0,0 +1,460 @@ +{ + "schemaVersion": 1, + "migrationType": "cosmos", + "configurationFingerprint": "f8044834808d5a3c6bf3579b7aee6ec71f503fb8a81edeab1e6399fc0a3cd0de", + "configuration": { + "sourceAccount": "thisismydemocosmos", + "sourceResourceGroup": "RG-Demos", + "sourceSubscriptionId": "9698dd71-9367-49c2-bede-fd0deecfad62", + "sourceDatabase": "SimpleChat", + "destinationAccount": "migration-test-sc", + "destinationResourceGroup": "RG-Demos", + "destinationSubscriptionId": "9698dd71-9367-49c2-bede-fd0deecfad62", + "destinationDatabase": "SimpleChat2", + "containers": [ + "activity_logs", + "agent_facts", + "agent_templates", + "approvals", + "archive_thoughts", + "archived_conversations", + "archived_messages", + "collaboration_conversations", + "collaboration_messages", + "collaboration_user_state", + "conversations", + "custom_pages", + "data_management_job_items", + "data_management_jobs", + "document_access_index", + "documents", + "feedback", + "file_processing", + "global_actions", + "global_agents", + "global_workspace_identities", + "governance_item_policies", + "governance_policies", + "group_actions", + "group_agents", + "group_conversations", + "group_documents", + "group_file_sync_items", + "group_file_sync_runs", + "group_file_sync_sources", + "group_messages", + "group_prompts", + "group_workflow_run_items", + "group_workflow_runs", + "group_workflows", + "group_workspace_identities", + "groups", + "messages", + "msgraph_pending_actions", + "notifications", + "personal_actions", + "personal_agents", + "personal_file_sync_items", + "personal_file_sync_runs", + "personal_file_sync_sources", + "personal_workflow_run_items", + "personal_workflow_runs", + "personal_workflows", + "personal_workspace_identities", + "prompts", + "public_documents", + "public_file_sync_items", + "public_file_sync_runs", + "public_file_sync_sources", + "public_prompts", + "public_workspace_identities", + "public_workspaces", + "safety", + "search_cache", + "service_throttle_logs", + "settings", + "tabular_export_runs", + "thoughts", + "user_settings", + "visual_image_runs", + "workflows" + ], + "containerSelectionMode": "all", + "mode": "differential", + "cosmosApiVersion": "2020-07-15", + "cosmosDnsSuffix": "documents.azure.com", + "excludedAdminSettingsDocument": "settings/app_settings" + }, + "status": "failed", + "createdUtc": "2026-07-21T20:27:51.6231634Z", + "updatedUtc": "2026-07-21T22:42:24.1751867Z", + "completedUtc": null, + "resumeCount": 3, + "currentResource": "document_access_index", + "lastError": "Document 'dai:user:441f7b4e-2f43-4a83-abf1-40697309b24d:personal:e6eead38-d25e-40b5-8ef7-70b5bd55fd23:1' in container 'document_access_index' failed after 5 attempt(s): HTTP 429. {\"code\":\"TooManyRequests\",\"message\":\"The request rate is too large. Please retry after sometime. Learn more: http://aka.ms/cosmosdb-error-429, Windows/10.0.20348 cosmos-netstandard-sdk/3.18.0\"}", + "resources": { + "public_file_sync_sources": { + "status": "completed", + "attempt": 1, + "startedUtc": "2026-07-21T20:27:52.1761467Z", + "updatedUtc": "2026-07-21T20:27:52.9363193Z", + "completedUtc": "2026-07-21T20:27:52.9363193Z", + "lastError": null, + "progress": {}, + "result": { + "CopiedCount": 0, + "SkippedCount": 0, + "ProcessedCount": 0, + "TotalCount": 0, + "RetryCount": 0 + } + }, + "personal_workflow_runs": { + "status": "completed", + "attempt": 1, + "startedUtc": "2026-07-21T20:27:52.9388451Z", + "updatedUtc": "2026-07-21T20:27:56.3127376Z", + "completedUtc": "2026-07-21T20:27:56.3127376Z", + "lastError": null, + "progress": {}, + "result": { + "CopiedCount": 35, + "SkippedCount": 0, + "ProcessedCount": 35, + "TotalCount": 35, + "RetryCount": 0 + } + }, + "data_management_jobs": { + "status": "completed", + "attempt": 1, + "startedUtc": "2026-07-21T20:27:56.3186547Z", + "updatedUtc": "2026-07-21T20:27:57.5023713Z", + "completedUtc": "2026-07-21T20:27:57.5023713Z", + "lastError": null, + "progress": {}, + "result": { + "CopiedCount": 3, + "SkippedCount": 0, + "ProcessedCount": 3, + "TotalCount": 3, + "RetryCount": 0 + } + }, + "global_workspace_identities": { + "status": "completed", + "attempt": 1, + "startedUtc": "2026-07-21T20:27:57.507741Z", + "updatedUtc": "2026-07-21T20:27:58.087601Z", + "completedUtc": "2026-07-21T20:27:58.087601Z", + "lastError": null, + "progress": {}, + "result": { + "CopiedCount": 0, + "SkippedCount": 0, + "ProcessedCount": 0, + "TotalCount": 0, + "RetryCount": 0 + } + }, + "visual_image_runs": { + "status": "completed", + "attempt": 1, + "startedUtc": "2026-07-21T20:27:58.0903395Z", + "updatedUtc": "2026-07-21T20:27:59.1706123Z", + "completedUtc": "2026-07-21T20:27:59.1706123Z", + "lastError": null, + "progress": {}, + "result": { + "CopiedCount": 3, + "SkippedCount": 0, + "ProcessedCount": 3, + "TotalCount": 3, + "RetryCount": 0 + } + }, + "public_documents": { + "status": "completed", + "attempt": 1, + "startedUtc": "2026-07-21T20:27:59.1754963Z", + "updatedUtc": "2026-07-21T20:28:02.0541212Z", + "completedUtc": "2026-07-21T20:28:02.0541212Z", + "lastError": null, + "progress": {}, + "result": { + "CopiedCount": 38, + "SkippedCount": 0, + "ProcessedCount": 38, + "TotalCount": 38, + "RetryCount": 0 + } + }, + "personal_file_sync_sources": { + "status": "completed", + "attempt": 1, + "startedUtc": "2026-07-21T20:28:02.0696031Z", + "updatedUtc": "2026-07-21T20:28:03.2515497Z", + "completedUtc": "2026-07-21T20:28:03.2515497Z", + "lastError": null, + "progress": {}, + "result": { + "CopiedCount": 2, + "SkippedCount": 0, + "ProcessedCount": 2, + "TotalCount": 2, + "RetryCount": 0 + } + }, + "data_management_job_items": { + "status": "completed", + "attempt": 1, + "startedUtc": "2026-07-21T20:28:03.2572852Z", + "updatedUtc": "2026-07-21T20:28:05.8860515Z", + "completedUtc": "2026-07-21T20:28:05.8860515Z", + "lastError": null, + "progress": {}, + "result": { + "CopiedCount": 34, + "SkippedCount": 0, + "ProcessedCount": 34, + "TotalCount": 34, + "RetryCount": 0 + } + }, + "prompts": { + "status": "completed", + "attempt": 1, + "startedUtc": "2026-07-21T20:28:05.8906866Z", + "updatedUtc": "2026-07-21T20:28:07.4371353Z", + "completedUtc": "2026-07-21T20:28:07.4371353Z", + "lastError": null, + "progress": {}, + "result": { + "CopiedCount": 16, + "SkippedCount": 0, + "ProcessedCount": 16, + "TotalCount": 16, + "RetryCount": 0 + } + }, + "group_workspace_identities": { + "status": "completed", + "attempt": 1, + "startedUtc": "2026-07-21T20:28:07.4419397Z", + "updatedUtc": "2026-07-21T20:28:08.3263897Z", + "completedUtc": "2026-07-21T20:28:08.3263897Z", + "lastError": null, + "progress": {}, + "result": { + "CopiedCount": 0, + "SkippedCount": 0, + "ProcessedCount": 0, + "TotalCount": 0, + "RetryCount": 0 + } + }, + "feedback": { + "status": "completed", + "attempt": 1, + "startedUtc": "2026-07-21T20:28:08.3286231Z", + "updatedUtc": "2026-07-21T20:28:09.7918773Z", + "completedUtc": "2026-07-21T20:28:09.7918773Z", + "lastError": null, + "progress": {}, + "result": { + "CopiedCount": 0, + "SkippedCount": 28, + "ProcessedCount": 28, + "TotalCount": 28, + "RetryCount": 0 + } + }, + "group_workflow_run_items": { + "status": "completed", + "attempt": 1, + "startedUtc": "2026-07-21T20:28:09.7970675Z", + "updatedUtc": "2026-07-21T20:28:10.3379742Z", + "completedUtc": "2026-07-21T20:28:10.3379742Z", + "lastError": null, + "progress": {}, + "result": { + "CopiedCount": 0, + "SkippedCount": 0, + "ProcessedCount": 0, + "TotalCount": 0, + "RetryCount": 0 + } + }, + "custom_pages": { + "status": "completed", + "attempt": 1, + "startedUtc": "2026-07-21T20:28:10.3404352Z", + "updatedUtc": "2026-07-21T20:28:11.3043477Z", + "completedUtc": "2026-07-21T20:28:11.3043477Z", + "lastError": null, + "progress": {}, + "result": { + "CopiedCount": 1, + "SkippedCount": 0, + "ProcessedCount": 1, + "TotalCount": 1, + "RetryCount": 0 + } + }, + "governance_policies": { + "status": "completed", + "attempt": 2, + "startedUtc": "2026-07-21T20:28:24.057504Z", + "updatedUtc": "2026-07-21T20:28:24.9581951Z", + "completedUtc": "2026-07-21T20:28:24.9581951Z", + "lastError": null, + "progress": {}, + "result": { + "CopiedCount": 10, + "SkippedCount": 0, + "ProcessedCount": 10, + "TotalCount": 10, + "RetryCount": 0 + } + }, + "public_file_sync_items": { + "status": "completed", + "attempt": 1, + "startedUtc": "2026-07-21T20:28:24.963358Z", + "updatedUtc": "2026-07-21T20:28:25.6901956Z", + "completedUtc": "2026-07-21T20:28:25.6901956Z", + "lastError": null, + "progress": {}, + "result": { + "CopiedCount": 0, + "SkippedCount": 0, + "ProcessedCount": 0, + "TotalCount": 0, + "RetryCount": 0 + } + }, + "group_workflow_runs": { + "status": "completed", + "attempt": 1, + "startedUtc": "2026-07-21T20:28:25.6926465Z", + "updatedUtc": "2026-07-21T20:28:26.2893584Z", + "completedUtc": "2026-07-21T20:28:26.2893584Z", + "lastError": null, + "progress": {}, + "result": { + "CopiedCount": 0, + "SkippedCount": 0, + "ProcessedCount": 0, + "TotalCount": 0, + "RetryCount": 0 + } + }, + "personal_actions": { + "status": "completed", + "attempt": 1, + "startedUtc": "2026-07-21T20:28:26.2916977Z", + "updatedUtc": "2026-07-21T20:28:28.9459563Z", + "completedUtc": "2026-07-21T20:28:28.9459563Z", + "lastError": null, + "progress": {}, + "result": { + "CopiedCount": 20, + "SkippedCount": 0, + "ProcessedCount": 20, + "TotalCount": 20, + "RetryCount": 4 + } + }, + "group_documents": { + "status": "completed", + "attempt": 1, + "startedUtc": "2026-07-21T20:28:28.9507122Z", + "updatedUtc": "2026-07-21T20:28:46.5712369Z", + "completedUtc": "2026-07-21T20:28:46.5712369Z", + "lastError": null, + "progress": {}, + "result": { + "CopiedCount": 352, + "SkippedCount": 0, + "ProcessedCount": 352, + "TotalCount": 352, + "RetryCount": 20 + } + }, + "public_workspace_identities": { + "status": "completed", + "attempt": 1, + "startedUtc": "2026-07-21T20:28:46.5763258Z", + "updatedUtc": "2026-07-21T20:28:47.2251468Z", + "completedUtc": "2026-07-21T20:28:47.2251468Z", + "lastError": null, + "progress": {}, + "result": { + "CopiedCount": 0, + "SkippedCount": 0, + "ProcessedCount": 0, + "TotalCount": 0, + "RetryCount": 0 + } + }, + "group_conversations": { + "status": "completed", + "attempt": 3, + "startedUtc": "2026-07-21T20:52:02.6996064Z", + "updatedUtc": "2026-07-21T22:42:09.6198737Z", + "completedUtc": "2026-07-21T22:42:09.6198737Z", + "lastError": null, + "progress": {}, + "result": { + "CopiedCount": 348035, + "SkippedCount": 9352, + "ProcessedCount": 357387, + "TotalCount": 357387, + "RetryCount": 69 + } + }, + "public_prompts": { + "status": "completed", + "attempt": 1, + "startedUtc": "2026-07-21T22:42:09.6316864Z", + "updatedUtc": "2026-07-21T22:42:15.0682058Z", + "completedUtc": "2026-07-21T22:42:15.0682058Z", + "lastError": null, + "progress": {}, + "result": { + "CopiedCount": 0, + "SkippedCount": 0, + "ProcessedCount": 0, + "TotalCount": 0, + "RetryCount": 0 + } + }, + "personal_workspace_identities": { + "status": "completed", + "attempt": 1, + "startedUtc": "2026-07-21T22:42:15.0716095Z", + "updatedUtc": "2026-07-21T22:42:18.1104476Z", + "completedUtc": "2026-07-21T22:42:18.1104476Z", + "lastError": null, + "progress": {}, + "result": { + "CopiedCount": 2, + "SkippedCount": 0, + "ProcessedCount": 2, + "TotalCount": 2, + "RetryCount": 0 + } + }, + "document_access_index": { + "status": "failed", + "attempt": 1, + "startedUtc": "2026-07-21T22:42:18.1173309Z", + "updatedUtc": "2026-07-21T22:42:24.1736431Z", + "completedUtc": null, + "lastError": "Document 'dai:user:441f7b4e-2f43-4a83-abf1-40697309b24d:personal:e6eead38-d25e-40b5-8ef7-70b5bd55fd23:1' in container 'document_access_index' failed after 5 attempt(s): HTTP 429. {\"code\":\"TooManyRequests\",\"message\":\"The request rate is too large. Please retry after sometime. Learn more: http://aka.ms/cosmosdb-error-429, Windows/10.0.20348 cosmos-netstandard-sdk/3.18.0\"}", + "progress": {}, + "result": {} + } + }, + "summary": {} +} diff --git a/ui_tests/test_chat_background_generated_export_status.py b/ui_tests/test_chat_background_generated_export_status.py index 0c762b7f1..3cdc83991 100644 --- a/ui_tests/test_chat_background_generated_export_status.py +++ b/ui_tests/test_chat_background_generated_export_status.py @@ -1,11 +1,11 @@ # test_chat_background_generated_export_status.py """ UI test for chat background generated export status cards. -Version: 0.241.046 -Implemented in: 0.241.046 +Version: 0.250.061 +Implemented in: 0.241.046; cancellation in 0.250.060; automatic-only refresh in 0.250.061 This test ensures queued tabular generated exports render progress in chat and -turn into a downloadable artifact when the status API reports completion. +turn into a downloadable artifact when complete or a visible canceled state. """ import os @@ -33,8 +33,8 @@ def _require_ui_env() -> None: @pytest.mark.ui -def test_chat_background_generated_export_status_card_refreshes_to_download(playwright) -> None: - """Validate queued background generated exports show progress and completion state.""" +def test_chat_background_generated_export_status_card_auto_refreshes_to_download(playwright) -> None: + """Validate queued exports transition automatically without a manual refresh control.""" _require_ui_env() browser = playwright.chromium.launch() @@ -120,12 +120,192 @@ def test_chat_background_generated_export_status_card_refreshes_to_download(play expect(message.get_by_text("Background export")).to_be_visible() expect(message.get_by_text("Running")).to_be_visible() expect(message.get_by_text("298 of 1,592 batches")).to_be_visible() - expect(message.get_by_role("button", name="Refresh Status")).to_be_visible() + expect(message.get_by_role("button", name="Refresh Status")).to_have_count(0) assert message.get_by_role("button", name="Download JSON").count() == 0 - message.get_by_role("button", name="Refresh Status").click() - expect(message.get_by_role("button", name="Download JSON")).to_be_visible() + expect(message.get_by_role("button", name="Download JSON")).to_be_visible(timeout=15000) expect(message.get_by_text("Saved to this chat for download in this conversation.")).to_be_visible() + finally: + context.close() + browser.close() + + +@pytest.mark.ui +def test_chat_background_generated_export_can_be_canceled(playwright) -> None: + """Validate a running export exposes Cancel and renders the durable canceled state.""" + _require_ui_env() + + browser = playwright.chromium.launch() + context = browser.new_context( + storage_state=STORAGE_STATE, + viewport={"width": 1440, "height": 900}, + ) + page = context.new_page() + page_errors = [] + cancel_requests = [] + page.on("pageerror", lambda error: page_errors.append(str(error))) + + try: + page.route( + "**/api/tabular/generated-output/runs/run-cancel-test/cancel", + lambda route: ( + cancel_requests.append(route.request.method), + route.fulfill( + status=200, + content_type="application/json", + json={ + "success": True, + "canceled": True, + "message": "Background export canceled.", + "run": { + "run_id": "run-cancel-test", + "status": "canceled", + "status_label": "Canceled", + "status_tone": "secondary", + "status_detail": "Export was canceled.", + "row_count": 30000, + "processed_rows": 1250, + "batch_count": 600, + "completed_batches": 25, + "progress_percent": 4.17, + "checkpoint_summary": "25 of 600 batches checkpointed; 1,250 of 30,000 rows processed", + "can_resume": False, + "can_cancel": False, + "background_export": True, + }, + }, + ), + ), + ) + page.goto(f"{BASE_URL}/", wait_until="domcontentloaded") + page.evaluate( + """ + async () => { + const module = await import('/static/js/chat/chat-messages.js'); + window.currentConversationId = 'conversation-cancel-test'; + module.appendMessage( + 'AI', + 'The exhaustive export is running.', + null, + 'message-cancel-test', + false, + [], + [], + [], + null, + null, + { + metadata: { + generated_tabular_outputs: [ + { + capability: 'tabular', + background_export: true, + export_run_id: 'run-cancel-test', + run_id: 'run-cancel-test', + status: 'running', + status_label: 'Running', + can_cancel: true, + can_resume: false, + file_name: 'generated-output.csv', + output_format: 'csv', + row_count: 30000, + processed_rows: 1250, + batch_count: 600, + completed_batches: 25, + source_file_name: 'large-source.csv' + } + ] + } + }, + false + ); + } + """ + ) + + message = page.locator('[data-message-id="message-cancel-test"]') + cancel_button = message.get_by_role("button", name="Cancel background export") + expect(cancel_button).to_be_visible() + cancel_button.click() + + expect(message.get_by_text("Canceled", exact=True)).to_be_visible() + expect(message.get_by_text("25 of 600 batches checkpointed")).to_be_visible() + expect(cancel_button).to_be_hidden() + assert cancel_requests == ["POST"] + assert page_errors == [] + finally: + context.close() + browser.close() + + +@pytest.mark.ui +def test_chat_failed_exhaustive_export_without_run_id_remains_visible(playwright) -> None: + """Validate terminal failure metadata renders even when queue creation never produced a run.""" + _require_ui_env() + + browser = playwright.chromium.launch() + context = browser.new_context( + storage_state=STORAGE_STATE, + viewport={"width": 1440, "height": 900}, + ) + page = context.new_page() + page_errors = [] + page.on("pageerror", lambda error: page_errors.append(str(error))) + + try: + page.goto(f"{BASE_URL}/", wait_until="domcontentloaded") + page.evaluate( + """ + async () => { + const module = await import('/static/js/chat/chat-messages.js'); + window.currentConversationId = 'conversation-failed-export'; + module.appendMessage( + 'AI', + 'The exhaustive export could not be prepared.', + null, + 'message-failed-export', + false, + [], + [], + [], + null, + null, + { + metadata: { + generated_tabular_outputs: [ + { + capability: 'tabular', + background_export: true, + status: 'failed', + status_label: 'Failed', + status_tone: 'danger', + status_detail: 'The source query could not be replayed. No partial CSV was created.', + suppress_assistant_table_export: true, + file_name: 'source_generated.json', + output_format: 'json', + row_count: 30000, + processed_rows: 0, + can_resume: false, + can_cancel: false + } + ] + } + }, + false + ); + } + """ + ) + + message = page.locator('[data-message-id="message-failed-export"]') + expect(message.get_by_text("Background export")).to_be_visible() + expect(message.get_by_text("Failed", exact=True)).to_be_visible() + expect(message.get_by_text("No partial CSV was created.")).to_be_visible() + expect(message.get_by_role("button", name="Continue")).to_have_count(0) + expect(message.get_by_role("button", name="Cancel background export")).to_have_count(0) + expect(message.get_by_role("button", name="Refresh Status")).to_have_count(0) + expect(message.get_by_role("button", name="Download JSON")).to_have_count(0) + assert page_errors == [] finally: context.close() browser.close() \ No newline at end of file From aea82a4b5ca1590ad08d9ef97c120ef0a75e18d7 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Wed, 22 Jul 2026 10:25:48 -0400 Subject: [PATCH 02/25] Add authorized mixed-source manifest contracts --- application/single_app/config.py | 2 +- .../functions_mixed_source_orchestration.py | 684 +++++++++++++++++ .../single_app/functions_search_service.py | 296 ++++++-- application/single_app/functions_settings.py | 6 + .../single_app/functions_workflow_runner.py | 49 +- application/single_app/route_backend_chats.py | 55 ++ ..._SOURCE_MANIFEST_AND_EVIDENCE_CONTRACTS.md | 115 +++ docs/explanation/release_notes.md | 10 + .../test_mixed_source_manifest_contracts.py | 694 ++++++++++++++++++ .../test_tabular_document_actions_workflow.py | 212 +++++- 10 files changed, 2049 insertions(+), 74 deletions(-) create mode 100644 application/single_app/functions_mixed_source_orchestration.py create mode 100644 docs/explanation/features/MIXED_SOURCE_MANIFEST_AND_EVIDENCE_CONTRACTS.md create mode 100644 functional_tests/test_mixed_source_manifest_contracts.py diff --git a/application/single_app/config.py b/application/single_app/config.py index a182549ff..7cd617af2 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -97,7 +97,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.250.061" +VERSION = "0.250.062" IS_DEVELOPMENT = is_development_env_enabled() SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax') diff --git a/application/single_app/functions_mixed_source_orchestration.py b/application/single_app/functions_mixed_source_orchestration.py new file mode 100644 index 000000000..11d4934ba --- /dev/null +++ b/application/single_app/functions_mixed_source_orchestration.py @@ -0,0 +1,684 @@ +# functions_mixed_source_orchestration.py +"""Authorization-safe source manifest and evidence contracts for mixed sources.""" + +import json +import logging +import math +import os +import time + +from functions_appinsights import log_event + + +SOURCE_KIND_TABULAR = "tabular" +SOURCE_KIND_NARRATIVE = "narrative" +SOURCE_KIND_UNSUPPORTED = "unsupported" +SOURCE_KIND_UNRESOLVED = "unresolved" +SOURCE_KINDS = frozenset({ + SOURCE_KIND_TABULAR, + SOURCE_KIND_NARRATIVE, + SOURCE_KIND_UNSUPPORTED, + SOURCE_KIND_UNRESOLVED, +}) + +SOURCE_SCOPE_PERSONAL = "personal" +SOURCE_SCOPE_GROUP = "group" +SOURCE_SCOPE_PUBLIC = "public" +SOURCE_SCOPE_CHAT = "chat" +SOURCE_SCOPES = frozenset({ + SOURCE_SCOPE_PERSONAL, + SOURCE_SCOPE_GROUP, + SOURCE_SCOPE_PUBLIC, + SOURCE_SCOPE_CHAT, +}) + +AUTHORIZATION_STATUS_AUTHORIZED = "authorized" +AUTHORIZATION_STATUS_UNRESOLVED = "unresolved" +SOURCE_MANIFEST_MAX_SOURCES = 100 + +SELECTION_MODE_SELECTED = "selected" +SELECTION_MODE_ALL = "all" +SELECTION_MODE_HISTORY = "history" +SELECTION_MODE_RELEVANCE = "relevance" +SELECTION_MODES = frozenset({ + SELECTION_MODE_SELECTED, + SELECTION_MODE_ALL, + SELECTION_MODE_HISTORY, + SELECTION_MODE_RELEVANCE, +}) + +TABULAR_SOURCE_EXTENSIONS = frozenset({".csv", ".xls", ".xlsx", ".xlsm"}) +NARRATIVE_SOURCE_EXTENSIONS = frozenset({ + ".txt", ".doc", ".docm", ".docx", ".html", ".htm", ".md", ".markdown", + ".json", ".xml", ".yaml", ".yml", ".log", ".pdf", ".ppt", ".pptx", + ".msg", ".vsdx", ".jpg", ".jpeg", ".png", ".bmp", ".tiff", ".tif", + ".heif", ".heic", ".3ga", ".aac", ".ac3", ".aif", ".aifc", ".aiff", + ".amr", ".ape", ".au", ".caf", ".dts", ".f4a", ".flac", ".m4a", + ".m4b", ".m4r", ".mka", ".mp2", ".mp3", ".mpa", ".oga", ".ogg", + ".opus", ".spx", ".wav", ".weba", ".wma", ".wv", ".mp4", ".mov", + ".avi", ".mkv", ".flv", ".mxf", ".gxf", ".ts", ".ps", ".3gp", + ".3gpp", ".mpg", ".wmv", ".asf", ".m4v", ".isma", ".ismv", + ".dvr-ms", ".webm", ".mpeg", +}) + +EVIDENCE_ENGINE_TABULAR_TOOLS = "tabular_tools" +EVIDENCE_ENGINE_DOCUMENT_ANALYSIS = "document_analysis" +EVIDENCE_ENGINE_HYBRID_SEARCH = "hybrid_search" +EVIDENCE_ENGINES = frozenset({ + EVIDENCE_ENGINE_TABULAR_TOOLS, + EVIDENCE_ENGINE_DOCUMENT_ANALYSIS, + EVIDENCE_ENGINE_HYBRID_SEARCH, +}) + +EVIDENCE_STATUS_COMPLETED = "completed" +EVIDENCE_STATUS_PARTIAL = "partial" +EVIDENCE_STATUS_FAILED = "failed" +EVIDENCE_STATUS_SKIPPED = "skipped" +EVIDENCE_STATUSES = frozenset({ + EVIDENCE_STATUS_COMPLETED, + EVIDENCE_STATUS_PARTIAL, + EVIDENCE_STATUS_FAILED, + EVIDENCE_STATUS_SKIPPED, +}) + +EVIDENCE_ENVELOPE_MAX_BYTES = 65536 +EVIDENCE_SUMMARY_MAX_BYTES = 4096 +EVIDENCE_ERROR_MAX_BYTES = 1024 +EVIDENCE_LIST_MAX_ITEMS = 10 +EVIDENCE_ITEM_MAX_BYTES = 1536 +EVIDENCE_COVERAGE_MAX_BYTES = 4096 +EVIDENCE_JSON_MAX_DEPTH = 4 +EVIDENCE_JSON_MAX_COLLECTION_ITEMS = 20 +EVIDENCE_JSON_MAX_STRING_BYTES = 1024 + + +def normalize_selection_mode(selection_mode, default=SELECTION_MODE_SELECTED): + """Return a supported selection mode or raise for an invalid explicit value.""" + normalized_default = str(default or "").strip().lower() + if normalized_default not in SELECTION_MODES: + raise ValueError("Invalid default selection_mode") + + normalized_mode = str(selection_mode or "").strip().lower() + if not normalized_mode: + return normalized_default + if normalized_mode not in SELECTION_MODES: + raise ValueError( + f"selection_mode must be one of: {', '.join(sorted(SELECTION_MODES))}" + ) + return normalized_mode + + +def classify_source_kind(file_name, document_item=None): + """Classify a resolved source by native capability without reading its content.""" + normalized_file_name = str(file_name or "").strip() + extension = os.path.splitext(normalized_file_name)[1].lower() + if extension in TABULAR_SOURCE_EXTENSIONS: + return SOURCE_KIND_TABULAR + if extension in NARRATIVE_SOURCE_EXTENSIONS: + return SOURCE_KIND_NARRATIVE + + document_item = document_item if isinstance(document_item, dict) else {} + if any( + document_item.get(field_name) + for field_name in ( + "num_file_chunks", + "comparison_text", + "extracted_text", + "vision_analysis", + ) + ): + return SOURCE_KIND_NARRATIVE + return SOURCE_KIND_UNSUPPORTED + + +def _normalize_document_id(requested_source): + if isinstance(requested_source, dict): + requested_source = requested_source.get("document_id") or requested_source.get("id") + return str(requested_source or "").strip() + + +def _normalize_identifier_list(values): + if values is None: + return [] + if isinstance(values, (str, int)): + values = [values] + return [ + normalized_value + for normalized_value in ( + str(value or "").strip() + for value in list(values) + ) + if normalized_value + ] + + +def _safe_file_name(file_name): + return str(file_name or "").replace("\\", "/").split("/")[-1].strip() + + +def _unresolved_manifest_entry(document_id): + return { + "document_id": document_id, + "display_name": None, + "file_name": None, + "extension": None, + "source_kind": SOURCE_KIND_UNRESOLVED, + "scope": None, + "scope_id": None, + "group_id": None, + "public_workspace_id": None, + "conversation_id": None, + "source_version": None, + "authorization_status": AUTHORIZATION_STATUS_UNRESOLVED, + } + + +def _build_authorized_manifest_entry(document_id, user_id, document_context): + if not isinstance(document_context, dict): + return _unresolved_manifest_entry(document_id) + + document_item = document_context.get("document") + if not isinstance(document_item, dict): + return _unresolved_manifest_entry(document_id) + + resolved_document_id = str(document_item.get("id") or "").strip() + if resolved_document_id != document_id: + return _unresolved_manifest_entry(document_id) + + scope = str(document_context.get("scope") or "").strip().lower() + if scope not in SOURCE_SCOPES: + return _unresolved_manifest_entry(document_id) + + group_id = None + public_workspace_id = None + conversation_id = str( + document_context.get("conversation_id") + or document_item.get("conversation_id") + or "" + ).strip() or None + + if scope == SOURCE_SCOPE_PERSONAL: + scope_id = str(document_item.get("user_id") or user_id or "").strip() + elif scope == SOURCE_SCOPE_GROUP: + group_id = str(document_context.get("group_id") or "").strip() or None + scope_id = group_id + elif scope == SOURCE_SCOPE_PUBLIC: + public_workspace_id = str( + document_context.get("public_workspace_id") or "" + ).strip() or None + scope_id = public_workspace_id + else: + scope_id = conversation_id + + if not scope_id: + return _unresolved_manifest_entry(document_id) + + file_name = _safe_file_name( + document_item.get("file_name") + or document_item.get("filename") + or document_item.get("title") + ) + display_name = str(document_item.get("title") or file_name or document_id).strip() + extension = os.path.splitext(file_name)[1].lower() or None + source_version = document_item.get("version") + if source_version is None: + source_version = document_item.get("source_version") + if source_version is not None and not isinstance(source_version, (str, int, float)): + source_version = str(source_version) + + return { + "document_id": document_id, + "display_name": display_name, + "file_name": file_name or None, + "extension": extension, + "source_kind": classify_source_kind(file_name, document_item=document_item), + "scope": scope, + "scope_id": scope_id, + "group_id": group_id, + "public_workspace_id": public_workspace_id, + "conversation_id": conversation_id, + "source_version": source_version, + "authorization_status": AUTHORIZATION_STATUS_AUTHORIZED, + } + + +def _default_document_context_batch_resolver(**resolver_arguments): + # Imported lazily so this contract module remains usable by startup code and isolated tests. + from functions_search_service import resolve_document_contexts + + resolver_arguments["include_content"] = False + return resolve_document_contexts(**resolver_arguments) + + +def resolve_authorized_source_manifest( + requested_sources, + user_id, + selection_mode=SELECTION_MODE_SELECTED, + conversation_id=None, + active_group_ids=None, + active_public_workspace_ids=None, + context_resolver=None, +): + """Resolve each unique requested ID once into an ordered, authorized manifest.""" + normalized_selection_mode = normalize_selection_mode(selection_mode) + normalized_user_id = str(user_id or "").strip() + if not normalized_user_id: + raise ValueError("user_id is required") + + if isinstance(requested_sources, (str, int, dict)): + requested_source_list = [requested_sources] + else: + requested_source_list = list(requested_sources or []) + if len(requested_source_list) > SOURCE_MANIFEST_MAX_SOURCES: + log_event( + "[MixedSourceManifest] Rejected over-limit source manifest request.", + extra={ + "selection_mode": normalized_selection_mode, + "requested_source_count": len(requested_source_list), + "source_limit": SOURCE_MANIFEST_MAX_SOURCES, + }, + level=logging.WARNING, + ) + raise ValueError( + f"A source manifest supports at most {SOURCE_MANIFEST_MAX_SOURCES} requested sources" + ) + + unique_document_ids = [] + seen_document_ids = set() + duplicate_ids_removed = 0 + for requested_source in requested_source_list: + document_id = _normalize_document_id(requested_source) + if not document_id: + continue + if document_id in seen_document_ids: + duplicate_ids_removed += 1 + continue + seen_document_ids.add(document_id) + unique_document_ids.append(document_id) + + if context_resolver is not None and not callable(context_resolver): + raise TypeError("context_resolver must be callable") + + started_at = time.perf_counter() + manifest = [] + resolution_error_count = 0 + normalized_active_group_ids = _normalize_identifier_list(active_group_ids) + normalized_public_workspace_ids = _normalize_identifier_list( + active_public_workspace_ids + ) + normalized_conversation_id = str(conversation_id or "").strip() or None + + resolved_contexts = None + if context_resolver is None: + try: + resolved_contexts = _default_document_context_batch_resolver( + document_ids=unique_document_ids, + user_id=normalized_user_id, + doc_scope="all", + active_group_ids=normalized_active_group_ids, + active_public_workspace_id=normalized_public_workspace_ids, + conversation_id=normalized_conversation_id, + ) + except Exception: + resolved_contexts = [None] * len(unique_document_ids) + resolution_error_count = len(unique_document_ids) + if ( + not isinstance(resolved_contexts, list) + or len(resolved_contexts) != len(unique_document_ids) + ): + resolved_contexts = [None] * len(unique_document_ids) + resolution_error_count = len(unique_document_ids) + + for document_index, document_id in enumerate(unique_document_ids): + document_context = None + if resolved_contexts is not None: + document_context = resolved_contexts[document_index] + else: + try: + document_context = context_resolver( + document_id=document_id, + user_id=normalized_user_id, + doc_scope="all", + active_group_ids=normalized_active_group_ids, + active_public_workspace_id=normalized_public_workspace_ids, + conversation_id=normalized_conversation_id, + ) + except Exception: + resolution_error_count += 1 + manifest.append( + _build_authorized_manifest_entry( + document_id, + normalized_user_id, + document_context, + ) + ) + + source_kind_counts = {source_kind: 0 for source_kind in SOURCE_KINDS} + scope_distribution = {scope: 0 for scope in SOURCE_SCOPES} + for entry in manifest: + source_kind_counts[entry["source_kind"]] += 1 + if entry["scope"] in scope_distribution: + scope_distribution[entry["scope"]] += 1 + + duration_ms = round((time.perf_counter() - started_at) * 1000, 3) + resolved_source_count = len(manifest) - source_kind_counts[SOURCE_KIND_UNRESOLVED] + log_event( + "[MixedSourceManifest] Resolved authorized source manifest.", + extra={ + "selection_mode": normalized_selection_mode, + "requested_source_count": len(requested_source_list), + "unique_source_count": len(unique_document_ids), + "resolved_source_count": resolved_source_count, + "tabular_source_count": source_kind_counts[SOURCE_KIND_TABULAR], + "narrative_source_count": source_kind_counts[SOURCE_KIND_NARRATIVE], + "unsupported_source_count": source_kind_counts[SOURCE_KIND_UNSUPPORTED], + "unresolved_or_unauthorized_count": source_kind_counts[SOURCE_KIND_UNRESOLVED], + "duplicate_ids_removed": duplicate_ids_removed, + "resolution_error_count": resolution_error_count, + "scope_distribution": scope_distribution, + "manifest_resolution_duration_ms": duration_ms, + }, + level=logging.INFO, + ) + return manifest + + +def partition_source_manifest(manifest): + """Partition a manifest by capability while preserving order within each cohort.""" + partitions = { + "tabular_sources": [], + "narrative_sources": [], + "unsupported_sources": [], + "unresolved_sources": [], + } + partition_key_by_source_kind = { + SOURCE_KIND_TABULAR: "tabular_sources", + SOURCE_KIND_NARRATIVE: "narrative_sources", + SOURCE_KIND_UNSUPPORTED: "unsupported_sources", + SOURCE_KIND_UNRESOLVED: "unresolved_sources", + } + + for raw_entry in list(manifest or []): + entry = raw_entry if isinstance(raw_entry, dict) else {} + document_id = _normalize_document_id(entry) + if entry.get("authorization_status") != AUTHORIZATION_STATUS_AUTHORIZED: + partitions["unresolved_sources"].append( + _unresolved_manifest_entry(document_id) + ) + continue + + partition_key = partition_key_by_source_kind.get( + entry.get("source_kind"), + "unsupported_sources", + ) + partitions[partition_key].append(entry) + + return partitions + + +def _truncate_utf8(value, max_bytes): + normalized_value = str(value or "") + encoded_value = normalized_value.encode("utf-8") + if len(encoded_value) <= max_bytes: + return normalized_value + if max_bytes <= 3: + return encoded_value[:max_bytes].decode("utf-8", errors="ignore") + return ( + encoded_value[:max_bytes - 3].decode("utf-8", errors="ignore").rstrip() + + "..." + ) + + +def _make_json_safe(value, depth=0): + if value is None or isinstance(value, (bool, int)): + return value, False + if isinstance(value, float): + return (value, False) if math.isfinite(value) else (None, True) + if isinstance(value, str): + bounded_value = _truncate_utf8(value, EVIDENCE_JSON_MAX_STRING_BYTES) + return bounded_value, bounded_value != value + if depth >= EVIDENCE_JSON_MAX_DEPTH: + return _truncate_utf8(str(value), EVIDENCE_JSON_MAX_STRING_BYTES), True + if isinstance(value, dict): + source_items = list(value.items()) + bounded_value = {} + was_truncated = len(source_items) > EVIDENCE_JSON_MAX_COLLECTION_ITEMS + for key, item_value in source_items[:EVIDENCE_JSON_MAX_COLLECTION_ITEMS]: + normalized_key = _truncate_utf8(key, 128) + bounded_item, item_was_truncated = _make_json_safe( + item_value, + depth + 1, + ) + if ( + not isinstance(key, str) + or normalized_key != key + or normalized_key in bounded_value + ): + was_truncated = True + bounded_value[normalized_key] = bounded_item + was_truncated = was_truncated or item_was_truncated + return bounded_value, was_truncated + if isinstance(value, (list, tuple, set)): + source_items = list(value) + bounded_value = [] + was_truncated = len(source_items) > EVIDENCE_JSON_MAX_COLLECTION_ITEMS + for item in source_items[:EVIDENCE_JSON_MAX_COLLECTION_ITEMS]: + bounded_item, item_was_truncated = _make_json_safe(item, depth + 1) + bounded_value.append(bounded_item) + was_truncated = was_truncated or item_was_truncated + return bounded_value, was_truncated + return _truncate_utf8(str(value), EVIDENCE_JSON_MAX_STRING_BYTES), True + + +def _json_size_bytes(value): + return len( + json.dumps( + value, + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + ).encode("utf-8") + ) + + +def _bound_json_value(value, max_bytes): + safe_value, safe_value_was_truncated = _make_json_safe(value) + if _json_size_bytes(safe_value) <= max_bytes: + return safe_value, safe_value_was_truncated + + serialized_preview = json.dumps( + safe_value, + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + ) + preview_max_bytes = max(16, max_bytes // 3) + while preview_max_bytes > 0: + bounded_value = { + "truncated": True, + "preview": _truncate_utf8(serialized_preview, preview_max_bytes), + } + if _json_size_bytes(bounded_value) <= max_bytes: + return bounded_value, True + preview_max_bytes //= 2 + return {"truncated": True}, True + + +def _bound_json_list(values): + if values is None: + return [], False + if not isinstance(values, (list, tuple)): + raise ValueError("Evidence collection values must be lists") + + source_values = list(values) + bounded_values = [] + was_truncated = len(source_values) > EVIDENCE_LIST_MAX_ITEMS + for value in source_values[:EVIDENCE_LIST_MAX_ITEMS]: + bounded_value, value_was_truncated = _bound_json_value( + value, + EVIDENCE_ITEM_MAX_BYTES, + ) + bounded_values.append(bounded_value) + was_truncated = was_truncated or value_was_truncated + return bounded_values, was_truncated + + +def _build_truncated_coverage(coverage, coverage_was_truncated=False): + normalized_coverage = dict(coverage or {}) + normalized_coverage["evidence_envelope_truncated"] = True + if coverage_was_truncated: + normalized_coverage["coverage_truncated"] = True + + bounded_coverage, additional_truncation = _bound_json_value( + normalized_coverage, + EVIDENCE_COVERAGE_MAX_BYTES, + ) + if additional_truncation: + return { + "evidence_envelope_truncated": True, + "coverage_truncated": True, + } + return bounded_coverage + + +def build_evidence_envelope( + document_id, + source_kind, + engine, + status, + summary="", + evidence=None, + citations=None, + generated_artifacts=None, + coverage=None, + error=None, +): + """Build a bounded, JSON-safe evidence envelope for later synthesis phases.""" + normalized_document_id = str(document_id or "").strip() + if not normalized_document_id: + raise ValueError("document_id is required") + + normalized_source_kind = str(source_kind or "").strip().lower() + if normalized_source_kind not in {SOURCE_KIND_TABULAR, SOURCE_KIND_NARRATIVE}: + raise ValueError("Evidence source_kind must be tabular or narrative") + + normalized_engine = str(engine or "").strip().lower() + if normalized_engine not in EVIDENCE_ENGINES: + raise ValueError(f"Unsupported evidence engine: {normalized_engine}") + + normalized_status = str(status or "").strip().lower() + if normalized_status not in EVIDENCE_STATUSES: + raise ValueError(f"Unsupported evidence status: {normalized_status}") + + if coverage is not None and not isinstance(coverage, dict): + raise ValueError("coverage must be a dictionary") + + bounded_evidence, evidence_was_truncated = _bound_json_list(evidence) + bounded_citations, citations_were_truncated = _bound_json_list(citations) + bounded_artifacts, artifacts_were_truncated = _bound_json_list(generated_artifacts) + normalized_summary = _truncate_utf8(summary, EVIDENCE_SUMMARY_MAX_BYTES) + normalized_error = ( + _truncate_utf8(error, EVIDENCE_ERROR_MAX_BYTES) + if error is not None + else None + ) + bounds_applied = bool( + evidence_was_truncated + or citations_were_truncated + or artifacts_were_truncated + or len(str(summary or "").encode("utf-8")) > EVIDENCE_SUMMARY_MAX_BYTES + or ( + error is not None + and len(str(error).encode("utf-8")) > EVIDENCE_ERROR_MAX_BYTES + ) + ) + normalized_coverage = dict(coverage or {}) + if bounds_applied: + normalized_coverage["evidence_envelope_truncated"] = True + bounded_coverage, coverage_was_truncated = _bound_json_value( + normalized_coverage, + EVIDENCE_COVERAGE_MAX_BYTES, + ) + if coverage_was_truncated: + bounded_coverage = _build_truncated_coverage( + {}, + coverage_was_truncated=True, + ) + + envelope = { + "document_id": normalized_document_id, + "source_kind": normalized_source_kind, + "engine": normalized_engine, + "status": normalized_status, + "summary": normalized_summary, + "evidence": bounded_evidence, + "citations": bounded_citations, + "generated_artifacts": bounded_artifacts, + "coverage": bounded_coverage, + "error": normalized_error, + } + + while _json_size_bytes(envelope) > EVIDENCE_ENVELOPE_MAX_BYTES: + candidate_field = max( + ("evidence", "citations", "generated_artifacts"), + key=lambda field_name: len(envelope[field_name]), + ) + if envelope[candidate_field]: + envelope[candidate_field].pop() + envelope["coverage"] = _build_truncated_coverage( + envelope["coverage"], + ) + continue + envelope["summary"] = _truncate_utf8( + envelope["summary"], + max(128, len(envelope["summary"].encode("utf-8")) // 2), + ) + if len(envelope["summary"].encode("utf-8")) <= 128: + raise ValueError("Unable to bound evidence envelope") + + return envelope + + +def serialize_evidence_envelope(envelope): + """Validate and serialize a bounded evidence envelope.""" + if not isinstance(envelope, dict): + raise ValueError("Evidence envelope must be a dictionary") + + required_fields = { + "document_id", + "source_kind", + "engine", + "status", + "summary", + "evidence", + "citations", + "generated_artifacts", + "coverage", + "error", + } + if set(envelope) != required_fields: + raise ValueError("Evidence envelope fields do not match the contract") + + bounded_envelope = build_evidence_envelope( + document_id=envelope["document_id"], + source_kind=envelope["source_kind"], + engine=envelope["engine"], + status=envelope["status"], + summary=envelope["summary"], + evidence=envelope["evidence"], + citations=envelope["citations"], + generated_artifacts=envelope["generated_artifacts"], + coverage=envelope["coverage"], + error=envelope["error"], + ) + + serialized_envelope = json.dumps( + bounded_envelope, + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + if len(serialized_envelope.encode("utf-8")) > EVIDENCE_ENVELOPE_MAX_BYTES: + raise ValueError("Evidence envelope exceeds its serialized size bound") + return serialized_envelope \ No newline at end of file diff --git a/application/single_app/functions_search_service.py b/application/single_app/functions_search_service.py index 58b9793b0..becbe4502 100644 --- a/application/single_app/functions_search_service.py +++ b/application/single_app/functions_search_service.py @@ -12,7 +12,12 @@ from azure.cosmos.exceptions import CosmosResourceNotFoundError from openai import AzureOpenAI -from config import CLIENTS, cognitive_services_scope, cosmos_messages_container +from config import ( + CLIENTS, + cognitive_services_scope, + cosmos_conversations_container, + cosmos_messages_container, +) from functions_appinsights import log_event from functions_debug import debug_print from functions_documents import get_document_record, get_ordered_document_chunks @@ -238,34 +243,101 @@ def _build_chat_upload_chunks(text_content, max_chunks=None): return chunks -def _resolve_chat_upload_context(document_id, conversation_id=None): +def _authorize_chat_upload_conversation(user_id, conversation_id): + normalized_user_id = str(user_id or "").strip() + normalized_conversation_id = str(conversation_id or "").strip() + if not normalized_user_id or not normalized_conversation_id: + return False + + try: + conversation_item = cosmos_conversations_container.read_item( + item=normalized_conversation_id, + partition_key=normalized_conversation_id, + ) + except CosmosResourceNotFoundError: + return False + except Exception as exc: + log_event( + "[SearchService] Failed to authorize chat upload conversation.", + extra={"exception_type": type(exc).__name__}, + level=logging.WARNING, + exceptionTraceback=True, + debug_only=True, + ) + return False + + return str(conversation_item.get("user_id") or "").strip() == normalized_user_id + + +def _resolve_chat_upload_context( + document_id, + user_id=None, + conversation_id=None, + include_content=True, + authorization_prechecked=False, +): normalized_conversation_id = str(conversation_id or "").strip() normalized_document_id = str(document_id or "").strip() if not normalized_conversation_id or not normalized_document_id: return None + if ( + not authorization_prechecked + and not _authorize_chat_upload_conversation(user_id, normalized_conversation_id) + ): + return None try: - message_item = cosmos_messages_container.read_item( - item=normalized_document_id, - partition_key=normalized_conversation_id, - ) + if include_content: + message_item = cosmos_messages_container.read_item( + item=normalized_document_id, + partition_key=normalized_conversation_id, + ) + else: + metadata_items = list(cosmos_messages_container.query_items( + query=""" + SELECT TOP 1 + c.id, + c.role, + c.filename, + c.title, + c.version, + c.metadata.is_user_upload AS is_user_upload, + c.metadata.is_generated_chat_artifact AS is_generated_chat_artifact, + c.metadata.generated_artifact_capability AS generated_artifact_capability, + c.metadata.generated_artifact_output_format AS generated_artifact_output_format + FROM c + WHERE c.id = @document_id + """, + parameters=[ + {"name": "@document_id", "value": normalized_document_id}, + ], + partition_key=normalized_conversation_id, + )) + if not metadata_items: + return None + message_item = metadata_items[0] except CosmosResourceNotFoundError: return None except Exception as exc: - debug_print( - "[SearchService] Failed to resolve chat upload context | " - f"document_id={normalized_document_id} | conversation_id={normalized_conversation_id} | error={exc}" + log_event( + "[SearchService] Failed to resolve authorized chat upload context.", + extra={"exception_type": type(exc).__name__}, + level=logging.WARNING, + exceptionTraceback=True, + debug_only=True, ) return None role_name = str(message_item.get("role") or "").strip().lower() metadata = message_item.get("metadata", {}) or {} - is_uploaded_image = role_name == "image" and bool((message_item.get("metadata") or {}).get("is_user_upload")) + is_uploaded_image = role_name == "image" and bool( + metadata.get("is_user_upload") or message_item.get("is_user_upload") + ) if role_name not in {"file", "image"} or (role_name == "image" and not is_uploaded_image): return None - comparison_text = _coerce_chat_upload_text(message_item) - if not comparison_text: + comparison_text = _coerce_chat_upload_text(message_item) if include_content else "" + if include_content and not comparison_text: return None message_title = str(message_item.get("filename") or message_item.get("title") or normalized_document_id).strip() or normalized_document_id @@ -275,11 +347,24 @@ def _resolve_chat_upload_context(document_id, conversation_id=None): "title": message_title, "conversation_id": normalized_conversation_id, "source_type": "chat_upload", - "source_subtype": "generated_chat_artifact" if metadata.get("is_generated_chat_artifact") else "chat_upload", - "artifact_capability": str(metadata.get("generated_artifact_capability") or "").strip().lower() or None, - "artifact_output_format": str(metadata.get("generated_artifact_output_format") or "").strip().lower() or None, - "comparison_text": comparison_text, + "source_subtype": "generated_chat_artifact" if ( + metadata.get("is_generated_chat_artifact") + or message_item.get("is_generated_chat_artifact") + ) else "chat_upload", + "artifact_capability": str( + metadata.get("generated_artifact_capability") + or message_item.get("generated_artifact_capability") + or "" + ).strip().lower() or None, + "artifact_output_format": str( + metadata.get("generated_artifact_output_format") + or message_item.get("generated_artifact_output_format") + or "" + ).strip().lower() or None, + "version": message_item.get("version"), } + if include_content: + resolved_document["comparison_text"] = comparison_text return { "scope": "chat", "group_id": None, @@ -289,6 +374,59 @@ def _resolve_chat_upload_context(document_id, conversation_id=None): } +def _resolve_personal_document_context(document_id, user_id): + personal_document = get_document_record( + user_id=user_id, + document_id=document_id, + ) + if not personal_document: + return None + return { + "scope": "personal", + "group_id": None, + "public_workspace_id": None, + "document": personal_document, + } + + +def _resolve_group_document_context(document_id, user_id, authorized_group_ids): + for group_id in authorized_group_ids or []: + group_document = get_document_record( + user_id=user_id, + document_id=document_id, + group_id=group_id, + ) + if group_document: + return { + "scope": "group", + "group_id": group_id, + "public_workspace_id": None, + "document": group_document, + } + return None + + +def _resolve_public_document_context( + document_id, + user_id, + authorized_public_workspace_ids, +): + for public_workspace_id in authorized_public_workspace_ids or []: + public_document = get_document_record( + user_id=user_id, + document_id=document_id, + public_workspace_id=public_workspace_id, + ) + if public_document: + return { + "scope": "public", + "group_id": None, + "public_workspace_id": public_workspace_id, + "document": public_document, + } + return None + + def resolve_document_context( document_id, user_id, @@ -296,59 +434,45 @@ def resolve_document_context( active_group_ids=None, active_public_workspace_id=None, conversation_id=None, + include_content=True, ): normalized_scope = normalize_search_scope(doc_scope) if normalized_scope in ("all", "personal"): - personal_document = get_document_record(user_id=user_id, document_id=document_id) - if personal_document: - return { - "scope": "personal", - "group_id": None, - "public_workspace_id": None, - "document": personal_document, - } + personal_context = _resolve_personal_document_context(document_id, user_id) + if personal_context: + return personal_context if normalized_scope in ("all", "group"): - for group_id in _resolve_active_group_ids( + group_context = _resolve_group_document_context( + document_id, user_id, - active_group_ids=active_group_ids, - fallback_to_memberships=True, - ): - group_document = get_document_record( - user_id=user_id, - document_id=document_id, - group_id=group_id, - ) - if group_document: - return { - "scope": "group", - "group_id": group_id, - "public_workspace_id": None, - "document": group_document, - } + _resolve_active_group_ids( + user_id, + active_group_ids=active_group_ids, + fallback_to_memberships=True, + ), + ) + if group_context: + return group_context if normalized_scope in ("all", "public"): - for public_workspace_id in _resolve_public_workspace_ids( + public_context = _resolve_public_document_context( + document_id, user_id, - active_public_workspace_id=active_public_workspace_id, - ): - public_document = get_document_record( - user_id=user_id, - document_id=document_id, - public_workspace_id=public_workspace_id, - ) - if public_document: - return { - "scope": "public", - "group_id": None, - "public_workspace_id": public_workspace_id, - "document": public_document, - } + _resolve_public_workspace_ids( + user_id, + active_public_workspace_id=active_public_workspace_id, + ), + ) + if public_context: + return public_context chat_upload_context = _resolve_chat_upload_context( document_id=document_id, + user_id=user_id, conversation_id=conversation_id, + include_content=include_content, ) if chat_upload_context: return chat_upload_context @@ -356,6 +480,68 @@ def resolve_document_context( return None +def resolve_document_contexts( + document_ids, + user_id, + doc_scope="all", + active_group_ids=None, + active_public_workspace_id=None, + conversation_id=None, + include_content=True, +): + """Resolve ordered document contexts using one current authorization snapshot.""" + normalized_scope = normalize_search_scope(doc_scope) + normalized_document_ids = normalize_search_id_list(document_ids) + authorized_group_ids = [] + if normalized_scope in ("all", "group"): + authorized_group_ids = _resolve_active_group_ids( + user_id, + active_group_ids=active_group_ids, + fallback_to_memberships=True, + ) + authorized_public_workspace_ids = [] + if normalized_scope in ("all", "public"): + authorized_public_workspace_ids = _resolve_public_workspace_ids( + user_id, + active_public_workspace_id=active_public_workspace_id, + ) + + normalized_conversation_id = str(conversation_id or "").strip() + chat_conversation_authorized = bool( + normalized_conversation_id + and _authorize_chat_upload_conversation(user_id, normalized_conversation_id) + ) + + resolved_contexts = [] + for document_id in normalized_document_ids: + document_context = None + if normalized_scope in ("all", "personal"): + document_context = _resolve_personal_document_context(document_id, user_id) + if not document_context and normalized_scope in ("all", "group"): + document_context = _resolve_group_document_context( + document_id, + user_id, + authorized_group_ids, + ) + if not document_context and normalized_scope in ("all", "public"): + document_context = _resolve_public_document_context( + document_id, + user_id, + authorized_public_workspace_ids, + ) + if not document_context and chat_conversation_authorized: + document_context = _resolve_chat_upload_context( + document_id=document_id, + user_id=user_id, + conversation_id=normalized_conversation_id, + include_content=include_content, + authorization_prechecked=True, + ) + resolved_contexts.append(document_context) + + return resolved_contexts + + def build_search_request( query, user_id, diff --git a/application/single_app/functions_settings.py b/application/single_app/functions_settings.py index d54d3824f..6db9782cf 100644 --- a/application/single_app/functions_settings.py +++ b/application/single_app/functions_settings.py @@ -210,6 +210,11 @@ def is_tabular_processing_enabled(settings): return bool((settings or {}).get('enable_enhanced_citations', False)) +def is_mixed_source_manifest_enabled(settings): + """Return whether Phase 1 mixed-source manifest diagnostics are enabled.""" + return bool((settings or {}).get('enable_mixed_source_manifest', False)) + + CHAT_FILE_UPLOAD_APP_ROLE = "ChatFileUploadUser" WORKFLOW_USER_APP_ROLE = "WorkflowUser" DOCUMENT_INTELLIGENCE_PDF_IMAGE_EXTRACTION_MODES = {"read", "layout", "auto"} @@ -759,6 +764,7 @@ def get_settings(use_cosmos=False, include_source=False): 'enable_fact_memory_plugin': True, 'enable_tabular_processing_plugin': False, 'enable_multi_agent_orchestration': False, + 'enable_mixed_source_manifest': False, 'max_rounds_per_agent': 1, 'workflow_max_auto_invoke_attempts': 60, 'enable_semantic_kernel': False, diff --git a/application/single_app/functions_workflow_runner.py b/application/single_app/functions_workflow_runner.py index aff2f3848..4087561fa 100644 --- a/application/single_app/functions_workflow_runner.py +++ b/application/single_app/functions_workflow_runner.py @@ -78,6 +78,7 @@ build_agent_citation_artifact_documents, make_json_serializable, ) +from functions_mixed_source_orchestration import resolve_authorized_source_manifest from model_endpoint_clients import ( MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI, ) @@ -88,7 +89,14 @@ from functions_search_service import resolve_document_context, search_documents from functions_search import normalize_search_id_list, normalize_search_scope, normalize_search_top_n from functions_simplechat_operations import upload_generated_analysis_artifact_for_current_user -from functions_settings import get_settings, get_user_settings, is_tabular_processing_enabled, normalize_model_endpoints, resolve_model_endpoint_foundry_scope +from functions_settings import ( + get_settings, + get_user_settings, + is_mixed_source_manifest_enabled, + is_tabular_processing_enabled, + normalize_model_endpoints, + resolve_model_endpoint_foundry_scope, +) from functions_source_review import ( URL_ACCESS_CONTEXT_WORKFLOW, compact_source_review_result_for_metadata, @@ -1562,7 +1570,7 @@ def _normalize_tabular_source_hint(scope): return 'workspace' -def _resolve_tabular_document_action_documents(action_config, user_id, conversation_id=''): +def _get_document_action_source_ids(action_config): action_config = action_config if isinstance(action_config, dict) else {} action_type = str(action_config.get('type') or '').strip().lower() @@ -1588,6 +1596,16 @@ def _resolve_tabular_document_action_documents(action_config, user_id, conversat document_ids.append(document_id) role_by_document_id[document_id] = 'right' + return document_ids, role_by_document_id + + +def _resolve_tabular_document_action_documents( + action_config, + user_id, + conversation_id='', +): + action_config = action_config if isinstance(action_config, dict) else {} + document_ids, role_by_document_id = _get_document_action_source_ids(action_config) if not document_ids: return [] @@ -2002,13 +2020,36 @@ def _maybe_execute_tabular_document_action( ): if action_type not in {DOCUMENT_ACTION_TYPE_ANALYZE, DOCUMENT_ACTION_TYPE_COMPARISON}: return None - if not callable(invoke_prompt) or not is_tabular_processing_enabled(settings): - return None user_id = str(workflow.get('user_id') or '').strip() if not user_id: return None + if is_mixed_source_manifest_enabled(settings): + requested_source_ids, _ = _get_document_action_source_ids(action_config) + if requested_source_ids: + try: + resolve_authorized_source_manifest( + requested_source_ids, + user_id=user_id, + selection_mode='selected', + conversation_id=conversation_id, + active_group_ids=action_config.get('active_group_ids'), + active_public_workspace_ids=action_config.get('active_public_workspace_id'), + ) + except Exception: + log_event( + '[MixedSourceManifest] Workflow shadow resolution failed.', + extra={ + 'requested_source_count': len(requested_source_ids), + 'selection_mode': 'selected', + }, + level=logging.WARNING, + ) + + if not callable(invoke_prompt) or not is_tabular_processing_enabled(settings): + return None + tabular_documents = _resolve_tabular_document_action_documents( action_config, user_id, diff --git a/application/single_app/route_backend_chats.py b/application/single_app/route_backend_chats.py index 495bd9475..03c967be5 100644 --- a/application/single_app/route_backend_chats.py +++ b/application/single_app/route_backend_chats.py @@ -37,6 +37,7 @@ build_model_endpoint_sync_chat_client, build_semantic_kernel_chat_service_for_model, ) +from functions_mixed_source_orchestration import resolve_authorized_source_manifest import builtins import asyncio, types import ast @@ -289,6 +290,44 @@ def _normalize_capability_action(document_action_type): return ASSIGNED_KNOWLEDGE_USER_ACTION_SEARCH +def _maybe_resolve_chat_source_manifest( + settings, + user_id, + conversation_id, + selected_document_ids, + scope_context, +): + if not is_mixed_source_manifest_enabled(settings): + return [] + + requested_source_ids = _normalize_conversation_task_document_ids( + selected_document_ids + ) + if not requested_source_ids: + return [] + + scope_context = scope_context if isinstance(scope_context, dict) else {} + try: + return resolve_authorized_source_manifest( + requested_source_ids, + user_id=user_id, + selection_mode='selected', + conversation_id=conversation_id, + active_group_ids=scope_context.get('active_group_ids'), + active_public_workspace_ids=scope_context.get('active_public_workspace_ids'), + ) + except Exception: + log_event( + '[MixedSourceManifest] Chat shadow resolution failed.', + extra={ + 'requested_source_count': len(requested_source_ids), + 'selection_mode': 'selected', + }, + level=logging.WARNING, + ) + return [] + + def _source_review_metadata_used(source_review_result): if not isinstance(source_review_result, dict): return False @@ -13527,6 +13566,14 @@ def result_requires_message_reload(result: Any) -> bool: selected_document_id = effective_selected_document_id document_scope = effective_document_scope + _maybe_resolve_chat_source_manifest( + settings, + user_id, + conversation_id, + effective_selected_document_ids, + scope_context, + ) + # Clear plugin invocations at start of message processing to ensure # each message only shows citations for tools executed during that specific interaction plugin_logger = get_plugin_logger() @@ -17199,6 +17246,14 @@ def build_streaming_capability_usage(): selected_document_id = effective_selected_document_id document_scope = effective_document_scope + _maybe_resolve_chat_source_manifest( + settings, + user_id, + conversation_id, + effective_selected_document_ids, + scope_context, + ) + # Determine chat type actual_chat_type = 'personal_single_user' if conversation_item.get('chat_type'): diff --git a/docs/explanation/features/MIXED_SOURCE_MANIFEST_AND_EVIDENCE_CONTRACTS.md b/docs/explanation/features/MIXED_SOURCE_MANIFEST_AND_EVIDENCE_CONTRACTS.md new file mode 100644 index 000000000..d5d7aec74 --- /dev/null +++ b/docs/explanation/features/MIXED_SOURCE_MANIFEST_AND_EVIDENCE_CONTRACTS.md @@ -0,0 +1,115 @@ +# Mixed-Source Manifest and Evidence Contracts + +Implemented in version: **0.250.062** + +GitHub issue: [#1056](https://github.com/microsoft/simplechat/issues/1056) + +Parent initiative: [#1055](https://github.com/microsoft/simplechat/issues/1055) + +## Overview + +SimpleChat now has a shared, authorization-safe contract for describing mixed document selections before any processing engine is selected. The ordered manifest classifies each authorized source as tabular, narrative, or unsupported, while sources that cannot be resolved or authorized receive the same scrubbed unresolved shape. + +This is Phase 1 of the mixed-source orchestration initiative. It establishes internal contracts and diagnostics without changing Chat, Search, Analyze, Compare, conversation follow-up, or rollout behavior. + +## Purpose + +The manifest removes the need for later orchestration phases to repeatedly resolve the same document IDs into incompatible shapes. Its pure partition helper also preserves valid tabular and narrative cohorts when another selected source is unsupported or unresolved. + +The bounded evidence envelope gives later native engines one JSON-safe result shape without placing exhaustive rows or unbounded content into synthesis context. + +## Dependencies + +- Existing personal document ownership checks in `functions_documents.get_document_record(...)` +- Existing group membership checks used by `functions_search_service.resolve_document_context(...)` +- Existing public workspace visibility checks in `functions_public_workspaces.py` +- Personal conversation ownership checks for chat-upload message resolution +- Structured telemetry through `functions_appinsights.log_event(...)` + +No new authorization model, tabular runner, export subsystem, route, database container, or persisted migration is introduced. + +## Technical Specifications + +### Architecture + +`functions_mixed_source_orchestration.py` provides four internal contracts: + +- `resolve_authorized_source_manifest(...)` resolves each unique requested document ID once, preserves first-occurrence order, and ignores caller-supplied scope or identity metadata. +- `partition_source_manifest(...)` returns independent tabular, narrative, unsupported, and unresolved cohorts while preserving order inside each cohort. +- `normalize_selection_mode(...)` validates `selected`, `all`, `history`, and `relevance` modes for later phases. +- `build_evidence_envelope(...)` and `serialize_evidence_envelope(...)` validate engine/status values and enforce deterministic item, string, collection, and serialized-size limits. + +Authorized manifest entries include normalized document identity, display/file names, extension, source kind, canonical scope and scope ID, applicable group/public/conversation IDs, source version when available, and authorization status. + +Unresolved and unauthorized requests are deliberately indistinguishable. Their entries retain only the caller-requested document ID and return null source metadata with `source_kind` and `authorization_status` set to `unresolved`. + +### Authorization Boundaries + +- Personal sources are returned only when the current user owns the document or has an existing approved share. +- Group source candidates are restricted to current group memberships before document lookup. +- Public source candidates are restricted to currently visible public workspaces before document lookup. +- Chat-upload metadata is queried only after the personal conversation record is loaded and its owner matches the current user. The manifest query projects identity, filename/title, version, role, and inert artifact capability fields without loading embedded file content, extracted text, vision output, or blob data. +- Requested scope, scope IDs, owner IDs, group IDs, public workspace IDs, and conversation IDs embedded in source payloads are not accepted as authorization decisions. + +Current group memberships, public workspace visibility, and chat conversation ownership are resolved once per manifest request and reused for its bounded document lookups. Authorization is still revalidated on every new manifest request. + +Manifest diagnostics contain aggregate counts, scope distribution, duplicate count, error count, and resolution duration only. They do not contain document IDs, filenames, content, blob paths, credentials, or raw configuration. + +### Evidence Bounds + +The evidence envelope has a maximum serialized size of 65,536 bytes. Summary text, error text, collection counts, individual structured values, nesting depth, and coverage metadata are bounded independently. When limits are applied, coverage records `evidence_envelope_truncated`; exhaustive output remains the responsibility of generated artifacts or durable checkpoints. + +Source manifests accept at most 100 requested entries. Over-limit requests fail before document resolution and emit count-only diagnostics; sources are never silently truncated. + +### API Endpoints + +No API endpoints are added or changed in Phase 1. + +### Configuration Options + +- `enable_mixed_source_manifest`: internal, default-off flag for producing shadow manifests in Chat and workflow requests. + +The flag is intentionally not exposed in the admin UI in this phase. Disabling it restores the previous caller path with no data rollback because manifests are request-scoped and not persisted. + +### File Structure + +- `application/single_app/functions_mixed_source_orchestration.py` +- `application/single_app/functions_search_service.py` +- `application/single_app/functions_workflow_runner.py` +- `application/single_app/route_backend_chats.py` +- `functional_tests/test_mixed_source_manifest_contracts.py` +- `functional_tests/test_tabular_document_actions_workflow.py` + +## Usage Instructions + +This phase has no user workflow or UI changes. Internal callers may enable `enable_mixed_source_manifest` to produce authorization-safe shadow manifests for selected Chat or workflow sources while legacy execution remains unchanged. + +Later phases can consume the shared partition and evidence contracts instead of resolving document IDs again. They must continue to reauthorize sources at the object boundary and must not treat a persisted manifest as proof of current access. + +## Testing and Validation + +- Executable functional coverage: `functional_tests/test_mixed_source_manifest_contracts.py` +- Updated workflow regression: `functional_tests/test_tabular_document_actions_workflow.py` +- Coverage includes PDF plus XLSX, DOCX plus CSV in both orders, duplicate IDs, duplicate filenames across scopes, unresolved and unsupported sources among valid sources, personal/group/public/chat authorization, authorization loss, ordering, partitioning, evidence serialization/bounds, selection modes, and privacy-safe diagnostics. +- Python compilation, editor diagnostics, broken-access-control checks, XSS checks, route-policy checks, and whitespace validation are part of the Phase 1 validation gate. + +## Performance Considerations + +- Duplicate requested IDs are removed before resolution, preserving the first occurrence. +- Each unique requested ID is looked up once within a request-scoped authorization snapshot. +- Requests are capped at 100 source entries before authorization or document reads begin. +- Classification uses normalized metadata and does not read source content. +- Evidence bounding occurs before serialization so synthesis payloads remain predictable. + +## Known Limitations + +- Phase 1 does not activate document retrieval from explicit Chat selections. +- Phase 1 does not run mixed Analyze engines or synthesize their outputs. +- Phase 1 does not implement cross-format Compare. +- Phase 1 does not persist or reuse source context across follow-up turns. +- Phase 1 does not enumerate an Analyze All Documents catalog. +- Rollout and native-engine behavior changes remain scoped to #1057 through #1061. + +## Related Version Updates + +- `application/single_app/config.py` was updated from **0.250.061** to **0.250.062** for #1056. \ No newline at end of file diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index 5a25204aa..4281f0087 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,16 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](/explanation/features/) and [Fixes by Version](/explanation/fixes/). +### **(v0.250.062)** + +#### New Features + +* **Authorized Mixed-Source Manifest and Evidence Contracts** + * Added one ordered, authorization-safe source manifest that classifies selected personal, group, public, and chat-upload documents as tabular, narrative, unsupported, or unresolved without exposing inaccessible source metadata. + * Added independent capability partitions, validated selection modes, bounded engine-neutral evidence envelopes, and aggregate privacy-safe diagnostics for later Chat, Search, Analyze, and Compare phases. + * Kept Phase 1 behavior-neutral through a default-off internal shadow-manifest flag; native mixed-source execution remains scoped to follow-up issues #1057-#1061. + * (Ref: microsoft/simplechat#1056, parent microsoft/simplechat#1055, `functions_mixed_source_orchestration.py`, `MIXED_SOURCE_MANIFEST_AND_EVIDENCE_CONTRACTS.md`) + ### **(v0.250.061)** #### User Interface Enhancements diff --git a/functional_tests/test_mixed_source_manifest_contracts.py b/functional_tests/test_mixed_source_manifest_contracts.py new file mode 100644 index 000000000..42b10cf55 --- /dev/null +++ b/functional_tests/test_mixed_source_manifest_contracts.py @@ -0,0 +1,694 @@ +#!/usr/bin/env python3 +# test_mixed_source_manifest_contracts.py +""" +Functional test for authorized mixed-source manifest and evidence contracts. +Version: 0.250.062 +Implemented in: 0.250.062 + +This test ensures Phase 1 of #1056 resolves requested sources once through +current authorization boundaries, preserves ordering, partitions mixed source +types, and bounds engine-neutral evidence without implementing #1057-#1061. +Parent initiative: #1055. +""" + +import importlib.util +import json +import sys +import types +from pathlib import Path + +from azure.cosmos.exceptions import CosmosResourceNotFoundError + + +REPO_ROOT = Path(__file__).resolve().parents[1] +APP_ROOT = REPO_ROOT / "application" / "single_app" +SEARCH_SERVICE_PATH = APP_ROOT / "functions_search_service.py" +sys.path.insert(0, str(APP_ROOT)) + +import functions_mixed_source_orchestration as orchestration + +ORIGINAL_ORCHESTRATION_LOG_EVENT = orchestration.log_event + + +def setup_module(module=None): + orchestration.log_event = lambda *args, **kwargs: None + + +def teardown_module(module=None): + orchestration.log_event = ORIGINAL_ORCHESTRATION_LOG_EVENT + + +class FakeItemContainer: + def __init__(self, items=None): + self.items = dict(items or {}) + self.read_calls = [] + self.query_calls = [] + + def read_item(self, item, partition_key): + self.read_calls.append((partition_key, item)) + key = (partition_key, item) + if key not in self.items: + raise CosmosResourceNotFoundError(status_code=404, message="Not found") + return dict(self.items[key]) + + def query_items(self, query, parameters, partition_key): + self.query_calls.append({ + "query": query, + "parameters": list(parameters or []), + "partition_key": partition_key, + }) + parameter_values = { + parameter.get("name"): parameter.get("value") + for parameter in list(parameters or []) + } + document_id = parameter_values.get("@document_id") + message_item = self.items.get((partition_key, document_id)) + if not message_item: + return [] + metadata = message_item.get("metadata", {}) or {} + return [{ + "id": message_item.get("id"), + "role": message_item.get("role"), + "filename": message_item.get("filename"), + "title": message_item.get("title"), + "version": message_item.get("version"), + "is_user_upload": metadata.get("is_user_upload"), + "is_generated_chat_artifact": metadata.get("is_generated_chat_artifact"), + "generated_artifact_capability": metadata.get("generated_artifact_capability"), + "generated_artifact_output_format": metadata.get("generated_artifact_output_format"), + }] + + +def _normalize_id_list(values): + if values is None: + return [] + if isinstance(values, str): + values = [values] + normalized_values = [] + for value in list(values): + normalized_value = str(value or "").strip() + if normalized_value and normalized_value not in normalized_values: + normalized_values.append(normalized_value) + return normalized_values + + +def load_isolated_search_service(): + config_stub = types.ModuleType("config") + config_stub.CLIENTS = {} + config_stub.cognitive_services_scope = "https://example.invalid/.default" + config_stub.cosmos_conversations_container = FakeItemContainer() + config_stub.cosmos_messages_container = FakeItemContainer() + + appinsights_stub = types.ModuleType("functions_appinsights") + appinsights_stub.log_event = lambda *args, **kwargs: None + + debug_stub = types.ModuleType("functions_debug") + debug_stub.debug_print = lambda *args, **kwargs: None + + documents_stub = types.ModuleType("functions_documents") + documents_stub.get_document_record = lambda **kwargs: None + documents_stub.get_ordered_document_chunks = lambda **kwargs: [] + + group_stub = types.ModuleType("functions_group") + group_stub.get_user_groups = lambda user_id: [] + + public_stub = types.ModuleType("functions_public_workspaces") + public_stub.get_user_visible_public_workspace_ids_from_settings = lambda user_id: [] + + search_stub = types.ModuleType("functions_search") + search_stub.SEARCH_DEFAULT_TOP_N = 12 + search_stub.SEARCH_MAX_TOP_N = 500 + search_stub.hybrid_search = lambda **kwargs: [] + search_stub.normalize_search_id_list = _normalize_id_list + search_stub.normalize_search_scope = ( + lambda value: str(value or "all").strip().lower() + if str(value or "all").strip().lower() in {"all", "personal", "group", "public"} + else "all" + ) + search_stub.normalize_search_top_n = ( + lambda value, default_value, max_value: default_value if value is None else int(value) + ) + + settings_stub = types.ModuleType("functions_settings") + settings_stub.get_settings = lambda: {} + settings_stub.get_user_settings = lambda user_id: {"settings": {}} + + module_stubs = { + "config": config_stub, + "functions_appinsights": appinsights_stub, + "functions_debug": debug_stub, + "functions_documents": documents_stub, + "functions_group": group_stub, + "functions_public_workspaces": public_stub, + "functions_search": search_stub, + "functions_settings": settings_stub, + } + previous_modules = { + module_name: sys.modules.get(module_name) + for module_name in module_stubs + } + sys.modules.update(module_stubs) + + try: + module_spec = importlib.util.spec_from_file_location( + "functions_search_service_mixed_source_test", + SEARCH_SERVICE_PATH, + ) + search_service = importlib.util.module_from_spec(module_spec) + module_spec.loader.exec_module(search_service) + finally: + for module_name, previous_module in previous_modules.items(): + if previous_module is None: + sys.modules.pop(module_name, None) + else: + sys.modules[module_name] = previous_module + + return search_service + + +def build_authorized_resolver_fixture(): + search_service = load_isolated_search_service() + state = { + "group_ids": {"group-a"}, + "public_workspace_ids": {"public-a"}, + "group_authorization_count": 0, + "public_authorization_count": 0, + } + personal_documents = { + "personal-pdf": { + "id": "personal-pdf", + "user_id": "user-1", + "title": "Personal report", + "file_name": "report.pdf", + "version": 3, + }, + "personal-xlsx": { + "id": "personal-xlsx", + "user_id": "user-1", + "title": "Personal workbook", + "file_name": "data.xlsx", + "version": 4, + }, + "personal-docx": { + "id": "personal-docx", + "user_id": "user-1", + "title": "Personal narrative", + "file_name": "narrative.docx", + "version": 1, + }, + "personal-csv": { + "id": "personal-csv", + "user_id": "user-1", + "title": "Personal data", + "file_name": "data.csv", + "version": 2, + }, + "personal-unsupported": { + "id": "personal-unsupported", + "user_id": "user-1", + "title": "Unsupported archive", + "file_name": "archive.zip", + }, + } + group_documents = { + ("group-a", "group-csv"): { + "id": "group-csv", + "group_id": "group-a", + "title": "Shared group data", + "file_name": "shared.csv", + "version": 5, + }, + } + public_documents = { + ("public-a", "public-csv"): { + "id": "public-csv", + "public_workspace_id": "public-a", + "title": "Shared public data", + "file_name": "shared.csv", + "version": 6, + }, + } + + def get_document_record(user_id, document_id, group_id=None, public_workspace_id=None): + if group_id is not None: + return group_documents.get((group_id, document_id)) + if public_workspace_id is not None: + return public_documents.get((public_workspace_id, document_id)) + document_item = personal_documents.get(document_id) + if document_item and document_item.get("user_id") == user_id: + return dict(document_item) + return None + + search_service.get_document_record = get_document_record + def get_user_groups(user_id): + state["group_authorization_count"] += 1 + return ( + [{"id": group_id} for group_id in sorted(state["group_ids"])] + if user_id == "user-1" + else [] + ) + + def get_visible_public_workspace_ids(user_id): + state["public_authorization_count"] += 1 + return ( + sorted(state["public_workspace_ids"]) + if user_id == "user-1" + else [] + ) + + search_service.get_user_groups = get_user_groups + search_service.get_user_visible_public_workspace_ids_from_settings = ( + get_visible_public_workspace_ids + ) + search_service.get_user_settings = lambda user_id: {"settings": {}} + search_service.cosmos_conversations_container = FakeItemContainer({ + ("conversation-1", "conversation-1"): { + "id": "conversation-1", + "user_id": "user-1", + }, + }) + search_service.cosmos_messages_container = FakeItemContainer({ + ("conversation-1", "chat-csv"): { + "id": "chat-csv", + "role": "file", + "filename": "chat.csv", + "file_content": "name,value\nalpha,1", + }, + }) + + resolver_calls = [] + + def resolver(**resolver_arguments): + resolver_calls.append(resolver_arguments["document_id"]) + return search_service.resolve_document_context(**resolver_arguments) + + return search_service, state, resolver, resolver_calls + + +def resolve_manifest(document_ids, resolver, user_id="user-1", conversation_id="conversation-1"): + return orchestration.resolve_authorized_source_manifest( + document_ids, + user_id=user_id, + conversation_id=conversation_id, + context_resolver=resolver, + ) + + +def test_mixed_classification_order_and_partition(): + _, _, resolver, _ = build_authorized_resolver_fixture() + + pdf_xlsx_manifest = resolve_manifest( + ["personal-pdf", "personal-xlsx"], + resolver, + ) + assert [entry["document_id"] for entry in pdf_xlsx_manifest] == [ + "personal-pdf", + "personal-xlsx", + ] + assert [entry["source_kind"] for entry in pdf_xlsx_manifest] == [ + "narrative", + "tabular", + ] + + for document_ids in ( + ["personal-docx", "personal-csv"], + ["personal-csv", "personal-docx"], + ): + manifest = resolve_manifest(document_ids, resolver) + assert [entry["document_id"] for entry in manifest] == document_ids + partitions = orchestration.partition_source_manifest(manifest) + assert [entry["document_id"] for entry in partitions["tabular_sources"]] == [ + "personal-csv", + ] + assert [entry["document_id"] for entry in partitions["narrative_sources"]] == [ + "personal-docx", + ] + + +def test_duplicates_and_cross_scope_filename_identity(): + _, _, resolver, resolver_calls = build_authorized_resolver_fixture() + manifest = resolve_manifest( + [ + "personal-xlsx", + "personal-xlsx", + "group-csv", + "public-csv", + ], + resolver, + ) + + assert resolver_calls.count("personal-xlsx") == 1 + assert [entry["document_id"] for entry in manifest] == [ + "personal-xlsx", + "group-csv", + "public-csv", + ] + duplicate_name_entries = [ + entry for entry in manifest if entry["file_name"] == "shared.csv" + ] + assert len(duplicate_name_entries) == 2 + assert { + (entry["scope"], entry["scope_id"], entry["document_id"]) + for entry in duplicate_name_entries + } == { + ("group", "group-a", "group-csv"), + ("public", "public-a", "public-csv"), + } + + +def test_unresolved_and_unsupported_do_not_erase_valid_sources(): + _, _, resolver, _ = build_authorized_resolver_fixture() + manifest = resolve_manifest( + ["personal-csv", "missing-source", "personal-unsupported", "personal-pdf"], + resolver, + ) + partitions = orchestration.partition_source_manifest(manifest) + + assert [entry["document_id"] for entry in manifest] == [ + "personal-csv", + "missing-source", + "personal-unsupported", + "personal-pdf", + ] + assert [entry["document_id"] for entry in partitions["tabular_sources"]] == [ + "personal-csv", + ] + assert [entry["document_id"] for entry in partitions["narrative_sources"]] == [ + "personal-pdf", + ] + assert [entry["document_id"] for entry in partitions["unsupported_sources"]] == [ + "personal-unsupported", + ] + assert [entry["document_id"] for entry in partitions["unresolved_sources"]] == [ + "missing-source", + ] + unresolved_entry = partitions["unresolved_sources"][0] + assert unresolved_entry["authorization_status"] == "unresolved" + assert unresolved_entry["file_name"] is None + assert unresolved_entry["scope"] is None + assert unresolved_entry["scope_id"] is None + + +def test_personal_group_public_and_chat_authorization(): + search_service, state, resolver, _ = build_authorized_resolver_fixture() + manifest = resolve_manifest( + ["personal-pdf", "group-csv", "public-csv", "chat-csv"], + resolver, + ) + assert [entry["scope"] for entry in manifest] == [ + "personal", + "group", + "public", + "chat", + ] + assert manifest[3]["conversation_id"] == "conversation-1" + assert manifest[3]["scope_id"] == "conversation-1" + + original_search_service_module = sys.modules.get("functions_search_service") + original_content_coercer = search_service._coerce_chat_upload_text + search_service.cosmos_messages_container.read_calls.clear() + search_service.cosmos_messages_container.query_calls.clear() + search_service._coerce_chat_upload_text = lambda message_item: (_ for _ in ()).throw( + AssertionError("Manifest resolution must not load chat-upload content") + ) + sys.modules["functions_search_service"] = search_service + try: + metadata_only_chat_manifest = orchestration.resolve_authorized_source_manifest( + ["chat-csv"], + user_id="user-1", + conversation_id="conversation-1", + ) + finally: + search_service._coerce_chat_upload_text = original_content_coercer + if original_search_service_module is None: + sys.modules.pop("functions_search_service", None) + else: + sys.modules["functions_search_service"] = original_search_service_module + assert metadata_only_chat_manifest[0]["source_kind"] == "tabular" + assert metadata_only_chat_manifest[0]["authorization_status"] == "authorized" + assert search_service.cosmos_messages_container.read_calls == [] + assert len(search_service.cosmos_messages_container.query_calls) == 1 + assert "c.file_content" not in search_service.cosmos_messages_container.query_calls[0]["query"] + assert "c.extracted_text" not in search_service.cosmos_messages_container.query_calls[0]["query"] + + caller_scope_payload = [{ + "document_id": "personal-pdf", + "scope": "public", + "public_workspace_id": "caller-controlled-workspace", + }] + caller_scope_manifest = resolve_manifest(caller_scope_payload, resolver) + assert caller_scope_manifest[0]["scope"] == "personal" + assert caller_scope_manifest[0]["public_workspace_id"] is None + + state["group_ids"].clear() + state["public_workspace_ids"].clear() + search_service.cosmos_conversations_container.items[ + ("conversation-1", "conversation-1") + ]["user_id"] = "different-user" + search_service.cosmos_messages_container.read_calls.clear() + search_service.cosmos_messages_container.query_calls.clear() + + authorization_loss_manifest = resolve_manifest( + ["group-csv", "public-csv", "chat-csv"], + resolver, + ) + assert all( + entry["source_kind"] == "unresolved" + and entry["authorization_status"] == "unresolved" + and entry["file_name"] is None + and entry["scope"] is None + for entry in authorization_loss_manifest + ) + assert search_service.cosmos_messages_container.read_calls == [] + assert search_service.cosmos_messages_container.query_calls == [] + + personal_authorization_loss = resolve_manifest( + ["personal-pdf"], + resolver, + user_id="different-user", + ) + assert personal_authorization_loss[0]["source_kind"] == "unresolved" + assert personal_authorization_loss[0]["display_name"] is None + + +def test_selection_mode_normalization(): + for selection_mode in ("selected", "all", "history", "relevance"): + assert orchestration.normalize_selection_mode( + f" {selection_mode.upper()} " + ) == selection_mode + assert orchestration.normalize_selection_mode(None) == "selected" + + try: + orchestration.normalize_selection_mode("everything") + except ValueError: + pass + else: + raise AssertionError("Invalid selection_mode must fail validation") + + +def test_batch_authorization_snapshot_and_source_limit(): + search_service, state, _, _ = build_authorized_resolver_fixture() + state["group_authorization_count"] = 0 + state["public_authorization_count"] = 0 + search_service.cosmos_conversations_container.read_calls.clear() + + original_search_service_module = sys.modules.get("functions_search_service") + sys.modules["functions_search_service"] = search_service + try: + manifest = orchestration.resolve_authorized_source_manifest( + ["personal-pdf", "group-csv", "public-csv", "chat-csv"], + user_id="user-1", + conversation_id="conversation-1", + ) + finally: + if original_search_service_module is None: + sys.modules.pop("functions_search_service", None) + else: + sys.modules["functions_search_service"] = original_search_service_module + + assert [entry["scope"] for entry in manifest] == [ + "personal", + "group", + "public", + "chat", + ] + assert state["group_authorization_count"] == 1 + assert state["public_authorization_count"] == 1 + assert search_service.cosmos_conversations_container.read_calls == [ + ("conversation-1", "conversation-1"), + ] + + over_limit_resolver_calls = [] + over_limit_sources = [ + f"source-{source_index}" + for source_index in range(orchestration.SOURCE_MANIFEST_MAX_SOURCES + 1) + ] + try: + orchestration.resolve_authorized_source_manifest( + over_limit_sources, + user_id="user-1", + context_resolver=lambda **kwargs: over_limit_resolver_calls.append(kwargs), + ) + except ValueError: + pass + else: + raise AssertionError("Over-limit source manifests must fail validation") + assert over_limit_resolver_calls == [] + + +def test_evidence_envelope_serialization_and_bounds(): + oversized_item = { + "rows": [ + {"column": "x" * 5000, "value": row_number} + for row_number in range(50) + ] + } + envelope = orchestration.build_evidence_envelope( + document_id="personal-xlsx", + source_kind="tabular", + engine="tabular_tools", + status="partial", + summary="s" * 20000, + evidence=[oversized_item for _ in range(25)], + citations=[oversized_item for _ in range(25)], + generated_artifacts=[oversized_item for _ in range(25)], + coverage={"requested_rows": 1000000, "processed_rows": 500000}, + error="e" * 5000, + ) + serialized_envelope = orchestration.serialize_evidence_envelope(envelope) + round_tripped_envelope = json.loads(serialized_envelope) + + assert len(serialized_envelope.encode("utf-8")) <= ( + orchestration.EVIDENCE_ENVELOPE_MAX_BYTES + ) + assert len(round_tripped_envelope["evidence"]) <= ( + orchestration.EVIDENCE_LIST_MAX_ITEMS + ) + assert len(round_tripped_envelope["citations"]) <= ( + orchestration.EVIDENCE_LIST_MAX_ITEMS + ) + assert len(round_tripped_envelope["generated_artifacts"]) <= ( + orchestration.EVIDENCE_LIST_MAX_ITEMS + ) + assert round_tripped_envelope["coverage"]["evidence_envelope_truncated"] is True + assert round_tripped_envelope["summary"].endswith("...") + assert round_tripped_envelope["error"].endswith("...") + + direct_envelope = { + "document_id": "personal-xlsx", + "source_kind": "tabular", + "engine": "tabular_tools", + "status": "completed", + "summary": "direct", + "evidence": [ + {"score": float("nan"), "value": item_number} + for item_number in range(orchestration.EVIDENCE_LIST_MAX_ITEMS + 5) + ], + "citations": [], + "generated_artifacts": [], + "coverage": {}, + "error": None, + } + direct_serialized = orchestration.serialize_evidence_envelope(direct_envelope) + direct_round_trip = json.loads(direct_serialized) + assert len(direct_round_trip["evidence"]) == orchestration.EVIDENCE_LIST_MAX_ITEMS + assert direct_round_trip["evidence"][0]["score"] is None + assert direct_round_trip["coverage"]["evidence_envelope_truncated"] is True + assert "NaN" not in direct_serialized + + nested_bound_envelope = orchestration.build_evidence_envelope( + document_id="personal-xlsx", + source_kind="tabular", + engine="tabular_tools", + status="completed", + evidence=[{ + "values": list( + range(orchestration.EVIDENCE_JSON_MAX_COLLECTION_ITEMS + 1) + ), + }], + coverage={"bounded": True}, + ) + assert len(nested_bound_envelope["evidence"][0]["values"]) == ( + orchestration.EVIDENCE_JSON_MAX_COLLECTION_ITEMS + ) + assert nested_bound_envelope["coverage"]["evidence_envelope_truncated"] is True + assert len(json.dumps(nested_bound_envelope["coverage"]).encode("utf-8")) <= ( + orchestration.EVIDENCE_COVERAGE_MAX_BYTES + ) + + try: + orchestration.serialize_evidence_envelope({ + **direct_envelope, + "extra_content": "not part of the contract", + }) + except ValueError: + pass + else: + raise AssertionError("Evidence serializer must reject undeclared fields") + + +def test_manifest_diagnostics_are_aggregate_only(): + _, _, resolver, _ = build_authorized_resolver_fixture() + captured_events = [] + original_log_event = orchestration.log_event + orchestration.log_event = lambda message, **kwargs: captured_events.append( + {"message": message, **kwargs} + ) + try: + resolve_manifest( + ["personal-pdf", "personal-pdf", "group-csv", "missing-source"], + resolver, + ) + finally: + orchestration.log_event = original_log_event + + assert len(captured_events) == 1 + diagnostics = captured_events[0]["extra"] + assert diagnostics["requested_source_count"] == 4 + assert diagnostics["unique_source_count"] == 3 + assert diagnostics["duplicate_ids_removed"] == 1 + assert diagnostics["narrative_source_count"] == 1 + assert diagnostics["tabular_source_count"] == 1 + assert diagnostics["unresolved_or_unauthorized_count"] == 1 + serialized_diagnostics = json.dumps(diagnostics, sort_keys=True) + for sensitive_value in ( + "personal-pdf", + "group-csv", + "missing-source", + "report.pdf", + "shared.csv", + "conversation-1", + ): + assert sensitive_value not in serialized_diagnostics + + +def run_tests(): + tests = [ + test_mixed_classification_order_and_partition, + test_duplicates_and_cross_scope_filename_identity, + test_unresolved_and_unsupported_do_not_erase_valid_sources, + test_personal_group_public_and_chat_authorization, + test_selection_mode_normalization, + test_batch_authorization_snapshot_and_source_limit, + test_evidence_envelope_serialization_and_bounds, + test_manifest_diagnostics_are_aggregate_only, + ] + results = [] + setup_module() + try: + for test in tests: + try: + test() + print(f"PASS {test.__name__}") + results.append(True) + except Exception as exc: + print(f"FAIL {test.__name__}: {exc}") + results.append(False) + finally: + teardown_module() + print(f"Results: {sum(results)}/{len(results)} tests passed") + return all(results) + + +if __name__ == "__main__": + raise SystemExit(0 if run_tests() else 1) \ No newline at end of file diff --git a/functional_tests/test_tabular_document_actions_workflow.py b/functional_tests/test_tabular_document_actions_workflow.py index e08f554ad..eed5c9140 100644 --- a/functional_tests/test_tabular_document_actions_workflow.py +++ b/functional_tests/test_tabular_document_actions_workflow.py @@ -2,21 +2,39 @@ # test_tabular_document_actions_workflow.py """ Functional test for tabular document-action workflow support. -Version: 0.241.038 -Implemented in: 0.241.038 +Version: 0.250.062 +Implemented in: 0.241.038; mixed-source manifest coverage added in 0.250.062 This test ensures tabular document actions reuse the shared tabular analysis path for Analyze and comparison workflows instead of relying only on the search-grounded chat path, including row-linked related-document evidence and -live tabular activity thoughts. +live tabular activity thoughts. It also ensures the Phase 1 contract from +#1056 preserves valid tabular sources in a mixed selection. Parent: #1055. """ +import ast +import logging from pathlib import Path +import sys import traceback ROOT = Path(__file__).resolve().parents[1] +APP_ROOT = ROOT / "application" / "single_app" WORKFLOW_RUNNER_FILE = ROOT / "application" / "single_app" / "functions_workflow_runner.py" +sys.path.insert(0, str(APP_ROOT)) + +import functions_mixed_source_orchestration as orchestration + +ORIGINAL_ORCHESTRATION_LOG_EVENT = orchestration.log_event + + +def setup_module(module=None): + orchestration.log_event = lambda *args, **kwargs: None + + +def teardown_module(module=None): + orchestration.log_event = ORIGINAL_ORCHESTRATION_LOG_EVENT def read_text(path: Path) -> str: @@ -37,6 +55,12 @@ def test_shared_tabular_document_action_helper_exists() -> None: assert 'def _resolve_tabular_document_action_documents(' in workflow_runner_content, ( "Expected functions_workflow_runner.py to resolve selected tabular documents before dispatching analysis or comparison." ) + assert 'resolve_authorized_source_manifest(' in workflow_runner_content, ( + "Expected workflow document actions to support the authorized Phase 1 source manifest." + ) + assert 'is_mixed_source_manifest_enabled(settings)' in workflow_runner_content, ( + "Expected Phase 1 workflow manifest production to remain behind its internal flag." + ) assert 'augment_tabular_invocations_with_related_document_evidence(' in workflow_runner_content, ( "Expected the shared helper to reuse row-linked related-document augmentation for tabular workflows." ) @@ -98,24 +122,184 @@ def test_tabular_document_actions_stream_live_activity() -> None: print("Tabular document-action live thought plumbing checks passed") +def test_mixed_sources_preserve_valid_tabular_partition() -> None: + print("Testing mixed-source tabular partition behavior...") + + source_records = { + "narrative-doc": { + "scope": "personal", + "document": { + "id": "narrative-doc", + "user_id": "user-1", + "title": "Narrative", + "file_name": "narrative.docx", + }, + }, + "tabular-doc": { + "scope": "personal", + "document": { + "id": "tabular-doc", + "user_id": "user-1", + "title": "Table", + "file_name": "table.csv", + }, + }, + } + resolver = lambda **kwargs: source_records.get(kwargs["document_id"]) + manifest = orchestration.resolve_authorized_source_manifest( + ["narrative-doc", "tabular-doc"], + user_id="user-1", + context_resolver=resolver, + ) + partitions = orchestration.partition_source_manifest(manifest) + + assert [entry["document_id"] for entry in manifest] == [ + "narrative-doc", + "tabular-doc", + ] + assert [entry["document_id"] for entry in partitions["narrative_sources"]] == [ + "narrative-doc", + ] + assert [entry["document_id"] for entry in partitions["tabular_sources"]] == [ + "tabular-doc", + ] + + print("Mixed-source tabular partition checks passed") + + +def test_manifest_flag_does_not_change_workflow_dispatch() -> None: + print("Testing workflow manifest flag behavior equivalence...") + + workflow_runner_tree = ast.parse(read_text(WORKFLOW_RUNNER_FILE)) + helper_node = next( + node + for node in workflow_runner_tree.body + if isinstance(node, ast.FunctionDef) + and node.name == "_maybe_execute_tabular_document_action" + ) + helper_module = ast.Module(body=[helper_node], type_ignores=[]) + ast.fix_missing_locations(helper_module) + + legacy_resolver_calls = [] + manifest_calls = [] + namespace = { + "DOCUMENT_ACTION_TYPE_ANALYZE": "analyze", + "DOCUMENT_ACTION_TYPE_COMPARISON": "comparison", + "is_tabular_processing_enabled": lambda settings: True, + "is_mixed_source_manifest_enabled": lambda settings: bool( + settings.get("enable_mixed_source_manifest") + ), + "_get_document_action_source_ids": lambda action_config: ( + list(action_config.get("document_ids") or []), + {}, + ), + "resolve_authorized_source_manifest": lambda *args, **kwargs: ( + manifest_calls.append((args, kwargs)) or [] + ), + "_resolve_tabular_document_action_documents": lambda *args, **kwargs: ( + legacy_resolver_calls.append((args, kwargs)) or [{"document_id": "table-1"}] + ), + "_resolve_tabular_document_action_model_name": lambda workflow, settings: "", + "log_event": lambda *args, **kwargs: None, + "logging": logging, + } + exec(compile(helper_module, str(WORKFLOW_RUNNER_FILE), "exec"), namespace) + helper = namespace["_maybe_execute_tabular_document_action"] + action_config = {"type": "analyze", "document_ids": ["table-1"]} + workflow = {"user_id": "user-1"} + + disabled_result = helper( + "analyze", + workflow, + action_config, + {"enable_mixed_source_manifest": False}, + conversation_id="conversation-1", + invoke_prompt=lambda *args, **kwargs: None, + ) + disabled_legacy_call = legacy_resolver_calls[-1] + assert manifest_calls == [] + + enabled_result = helper( + "analyze", + workflow, + action_config, + {"enable_mixed_source_manifest": True}, + conversation_id="conversation-1", + invoke_prompt=lambda *args, **kwargs: None, + ) + enabled_legacy_call = legacy_resolver_calls[-1] + + assert disabled_result == enabled_result is None + assert disabled_legacy_call == enabled_legacy_call + assert len(manifest_calls) == 1 + assert manifest_calls[0][0] == (["table-1"],) + + namespace["is_tabular_processing_enabled"] = lambda settings: False + manifest_calls.clear() + legacy_resolver_calls.clear() + disabled_tabular_result = helper( + "analyze", + workflow, + action_config, + {"enable_mixed_source_manifest": True}, + conversation_id="conversation-1", + invoke_prompt=lambda *args, **kwargs: None, + ) + assert disabled_tabular_result is None + assert len(manifest_calls) == 1 + assert legacy_resolver_calls == [] + + print("Workflow manifest flag behavior equivalence checks passed") + + +def test_document_action_chat_does_not_duplicate_shadow_manifest() -> None: + print("Testing document-action Chat manifest ownership...") + + route_tree = ast.parse( + read_text(ROOT / "application" / "single_app" / "route_backend_chats.py") + ) + document_action_function = next( + node + for node in ast.walk(route_tree) + if isinstance(node, ast.FunctionDef) + and node.name == "execute_document_action_chat_request" + ) + manifest_calls = [ + node + for node in ast.walk(document_action_function) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Name) + and node.func.id == "_maybe_resolve_chat_source_manifest" + ] + assert manifest_calls == [] + + print("Document-action Chat manifest ownership checks passed") + + def run_tests() -> bool: tests = [ test_shared_tabular_document_action_helper_exists, test_analyze_and_compare_dispatch_use_tabular_helper, test_tabular_document_actions_stream_live_activity, + test_mixed_sources_preserve_valid_tabular_partition, + test_manifest_flag_does_not_change_workflow_dispatch, + test_document_action_chat_does_not_duplicate_shadow_manifest, ] results = [] - - for test in tests: - print(f"\nRunning {test.__name__}...") - try: - test() - print("PASS") - results.append(True) - except Exception as exc: - print(f"FAIL: {exc}") - traceback.print_exc() - results.append(False) + setup_module() + try: + for test in tests: + print(f"\nRunning {test.__name__}...") + try: + test() + print("PASS") + results.append(True) + except Exception as exc: + print(f"FAIL: {exc}") + traceback.print_exc() + results.append(False) + finally: + teardown_module() success = all(results) print(f"\nResults: {sum(results)}/{len(results)} tests passed") From 778c10fa0a47b4f0393a196e6f674dd1b9e089a3 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Wed, 22 Jul 2026 15:29:19 -0400 Subject: [PATCH 03/25] Add mixed-source chat search and migration recovery --- application/single_app/config.py | 2 +- .../single_app/foundry_agent_runtime.py | 15 +- .../functions_mixed_source_orchestration.py | 517 +++- .../single_app/functions_search_service.py | 76 +- application/single_app/functions_settings.py | 15 + .../single_app/functions_tabular_analysis.py | 6 + .../single_app/functions_workflow_runner.py | 336 ++- application/single_app/route_backend_chats.py | 1239 +++++++++- .../single_app/route_backend_collaboration.py | 2 + .../single_app/route_backend_conversations.py | 75 +- .../tabular_processing_plugin.py | 17 + .../static/js/chat/chat-messages.js | 13 +- application/single_app/templates/chats.html | 1 + ...IXED_SOURCE_CHAT_AND_SEARCH_CONSISTENCY.md | 164 ++ ...ON_JSON_PROPERTY_AND_SKIP_REPORTING_FIX.md | 65 + docs/explanation/release_notes.md | 17 + ...osmos_migration_document_skip_reporting.py | 392 +++ ...st_mixed_source_chat_search_consistency.py | 807 +++++++ .../test_mixed_source_manifest_contracts.py | 281 ++- scripts/Migration-AISearch.ps1 | 1308 ++++++++++ scripts/Migration-Cosmos.ps1 | 2152 +++++++++++++++++ scripts/Migration-State.ps1 | 323 +++ scripts/Migration-StorageAccount.ps1 | 315 +++ ...est_chat_mixed_source_selection_payload.py | 195 ++ 24 files changed, 8188 insertions(+), 145 deletions(-) create mode 100644 docs/explanation/features/MIXED_SOURCE_CHAT_AND_SEARCH_CONSISTENCY.md create mode 100644 docs/explanation/fixes/COSMOS_MIGRATION_JSON_PROPERTY_AND_SKIP_REPORTING_FIX.md create mode 100644 functional_tests/test_cosmos_migration_document_skip_reporting.py create mode 100644 functional_tests/test_mixed_source_chat_search_consistency.py create mode 100644 scripts/Migration-AISearch.ps1 create mode 100644 scripts/Migration-Cosmos.ps1 create mode 100644 scripts/Migration-State.ps1 create mode 100644 scripts/Migration-StorageAccount.ps1 create mode 100644 ui_tests/test_chat_mixed_source_selection_payload.py diff --git a/application/single_app/config.py b/application/single_app/config.py index 7cd617af2..b84e4bce5 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -97,7 +97,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.250.062" +VERSION = "0.250.064" IS_DEVELOPMENT = is_development_env_enabled() SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax') diff --git a/application/single_app/foundry_agent_runtime.py b/application/single_app/foundry_agent_runtime.py index 56189993e..4568f75a5 100644 --- a/application/single_app/foundry_agent_runtime.py +++ b/application/single_app/foundry_agent_runtime.py @@ -42,6 +42,8 @@ FOUNDRY_INTERNAL_METADATA_KEYS = { "active_group_ids", "active_public_workspace_ids", + "document_context_requested", + "selection_mode", "selected_document_ids", } FOUNDRY_FILE_SEARCHABLE_CONTEXT_MAX_CHARS = 6000 @@ -1222,6 +1224,9 @@ def _looks_like_document_context_message(text: str) -> bool: "chat-uploaded file", "selected document", "tabular analysis", + "mixed-source evidence handoff", + "evidence_envelopes", + "[workflow document search context]", ) return any(marker in normalized for marker in markers) @@ -1237,7 +1242,13 @@ def _build_foundry_workflow_input_text( if not text: continue if not include_document_context and _looks_like_document_context_message(text): - continue + workflow_task_marker = "[Workflow task]" + workflow_task_index = text.rfind(workflow_task_marker) + if workflow_task_index < 0: + continue + text = text[workflow_task_index + len(workflow_task_marker):].strip() + if not text: + continue role_value = getattr(message, "role", "user") role = str(role_value).strip().lower() or "user" if role.startswith("authorrole."): @@ -1800,6 +1811,8 @@ def _collect_foundry_response_file_inputs( foundry_settings: Dict[str, Any], metadata: Dict[str, Any], ) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]: + if not _coerce_bool(foundry_settings.get("include_document_context"), True): + return [], [] if not _coerce_bool(foundry_settings.get("include_file_inputs"), True): return [], [] diff --git a/application/single_app/functions_mixed_source_orchestration.py b/application/single_app/functions_mixed_source_orchestration.py index 11d4934ba..ccf9806f5 100644 --- a/application/single_app/functions_mixed_source_orchestration.py +++ b/application/single_app/functions_mixed_source_orchestration.py @@ -90,6 +90,8 @@ EVIDENCE_JSON_MAX_DEPTH = 4 EVIDENCE_JSON_MAX_COLLECTION_ITEMS = 20 EVIDENCE_JSON_MAX_STRING_BYTES = 1024 +MIXED_SOURCE_HANDOFF_MAX_BYTES = 49152 +MIXED_SOURCE_HANDOFF_MAX_ENVELOPES = 20 def normalize_selection_mode(selection_mode, default=SELECTION_MODE_SELECTED): @@ -108,6 +110,146 @@ def normalize_selection_mode(selection_mode, default=SELECTION_MODE_SELECTED): return normalized_mode +def normalize_document_context_request( + selection_mode=None, + selected_document_ids=None, + document_context_requested=None, + hybrid_search=False, +): + """Validate the Phase 2 request contract and derive effective context intent.""" + normalized_document_ids = [] + seen_document_ids = set() + for document_id in _normalize_identifier_list(selected_document_ids): + if document_id == "all" or document_id in seen_document_ids: + continue + seen_document_ids.add(document_id) + normalized_document_ids.append(document_id) + + normalized_hybrid_search = _normalize_boolean( + hybrid_search, + field_name="hybrid_search", + ) + normalized_context_requested = _normalize_optional_boolean( + document_context_requested, + field_name="document_context_requested", + ) + has_explicit_selection_mode = str(selection_mode or "").strip() != "" + + if normalized_document_ids: + normalized_selection_mode = normalize_selection_mode( + selection_mode, + default=SELECTION_MODE_SELECTED, + ) + if normalized_selection_mode != SELECTION_MODE_SELECTED: + raise ValueError( + "selection_mode must be selected when selected_document_ids are provided" + ) + normalized_context_requested = True + elif has_explicit_selection_mode: + normalized_selection_mode = normalize_selection_mode(selection_mode) + if normalized_selection_mode == SELECTION_MODE_SELECTED: + raise ValueError( + "selection_mode selected requires at least one selected_document_id" + ) + if ( + normalized_selection_mode in {SELECTION_MODE_ALL, SELECTION_MODE_RELEVANCE} + and normalized_context_requested is None + ): + normalized_context_requested = True + elif normalized_context_requested is True or normalized_hybrid_search: + normalized_selection_mode = SELECTION_MODE_RELEVANCE + normalized_context_requested = True + else: + normalized_selection_mode = None + normalized_context_requested = False + + return { + "selection_mode": normalized_selection_mode, + "selected_document_ids": normalized_document_ids, + "document_context_requested": bool(normalized_context_requested), + "hybrid_search": normalized_hybrid_search, + "explicit_selection": bool(normalized_document_ids), + } + + +def should_run_tabular_evidence(user_question, has_narrative_sources=False): + """Return whether a mixed-source question needs tabular data or schema evidence.""" + normalized_question = " ".join(str(user_question or "").strip().lower().split()) + if not normalized_question: + return True + + tabular_markers = ( + "calculate", "calculation", "count", "average", "mean", "median", + "minimum", "maximum", "total", "sum", "percentage", "percent", + "rows", "columns", "spreadsheet", "workbook", "worksheet", "sheet", + "csv", "xlsx", "xls", "table", "tabular", "data set", "dataset", + "trend", "group by", "how many", "highest", "lowest", + ) + collective_markers = ( + "both files", "both documents", "all files", "all documents", + "all selected", "each file", "each document", "each source", + "across the files", "across the documents", "across the sources", + "mixed sources", + ) + narrative_markers = ( + "pdf", "docx", "word document", "presentation", "powerpoint", + "paragraph", "section", "policy", "procedure", "contract", + "agreement", "memo", "letter", "narrative", "prose", "report", + ) + + if any(marker in normalized_question for marker in tabular_markers): + return True + if any(marker in normalized_question for marker in collective_markers): + return True + if has_narrative_sources and any( + marker in normalized_question for marker in narrative_markers + ): + return False + if normalized_question in {"summarize", "summary", "summarize the selected sources"}: + return True + if has_narrative_sources: + return False + return True + + +def build_tabular_file_contexts_from_manifest(tabular_sources): + """Build canonical per-file contexts for the existing tabular runner.""" + contexts = [] + seen_source_identities = set() + for raw_source in list(tabular_sources or []): + source = raw_source if isinstance(raw_source, dict) else {} + if ( + source.get("authorization_status") != AUTHORIZATION_STATUS_AUTHORIZED + or source.get("source_kind") != SOURCE_KIND_TABULAR + ): + continue + + document_id = str(source.get("document_id") or "").strip() + file_name = str(source.get("file_name") or "").strip() + scope = str(source.get("scope") or "").strip().lower() + if not document_id or not file_name or scope not in SOURCE_SCOPES: + continue + source_hint = "workspace" if scope == SOURCE_SCOPE_PERSONAL else scope + source_identity = ( + document_id, + source_hint, + str(source.get("scope_id") or "").strip(), + ) + if source_identity in seen_source_identities: + continue + seen_source_identities.add(source_identity) + contexts.append({ + "document_id": document_id, + "file_name": file_name, + "source_hint": source_hint, + "group_id": source.get("group_id"), + "public_workspace_id": source.get("public_workspace_id"), + "conversation_id": source.get("conversation_id"), + "storage_locator": dict(source.get("storage_locator") or {}), + }) + return contexts + + def classify_source_kind(file_name, document_item=None): """Classify a resolved source by native capability without reading its content.""" normalized_file_name = str(file_name or "").strip() @@ -152,6 +294,28 @@ def _normalize_identifier_list(values): ] +def _normalize_boolean(value, field_name): + if isinstance(value, bool): + return value + if value in (None, ""): + return False + if isinstance(value, int) and value in (0, 1): + return bool(value) + if isinstance(value, str): + normalized_value = value.strip().lower() + if normalized_value in {"true", "1"}: + return True + if normalized_value in {"false", "0"}: + return False + raise ValueError(f"{field_name} must be a boolean") + + +def _normalize_optional_boolean(value, field_name): + if value is None or value == "": + return None + return _normalize_boolean(value, field_name=field_name) + + def _safe_file_name(file_name): return str(file_name or "").replace("\\", "/").split("/")[-1].strip() @@ -169,6 +333,7 @@ def _unresolved_manifest_entry(document_id): "public_workspace_id": None, "conversation_id": None, "source_version": None, + "storage_locator": None, "authorization_status": AUTHORIZATION_STATUS_UNRESOLVED, } @@ -199,9 +364,20 @@ def _build_authorized_manifest_entry(document_id, user_id, document_context): if scope == SOURCE_SCOPE_PERSONAL: scope_id = str(document_item.get("user_id") or user_id or "").strip() + if scope_id != str(user_id or "").strip() and not any( + str(shared_entry or "").strip() == f"{user_id},approved" + for shared_entry in document_item.get("shared_user_ids", []) or [] + ): + return _unresolved_manifest_entry(document_id) elif scope == SOURCE_SCOPE_GROUP: group_id = str(document_context.get("group_id") or "").strip() or None scope_id = group_id + document_group_id = str(document_item.get("group_id") or "").strip() + if document_group_id != group_id and not any( + str(shared_entry or "").strip() == f"{group_id},approved" + for shared_entry in document_item.get("shared_group_ids", []) or [] + ): + return _unresolved_manifest_entry(document_id) elif scope == SOURCE_SCOPE_PUBLIC: public_workspace_id = str( document_context.get("public_workspace_id") or "" @@ -226,6 +402,37 @@ def _build_authorized_manifest_entry(document_id, user_id, document_context): if source_version is not None and not isinstance(source_version, (str, int, float)): source_version = str(source_version) + storage_locator = None + if scope != SOURCE_SCOPE_CHAT: + try: + from functions_documents import get_document_blob_storage_info + + blob_container, blob_path = get_document_blob_storage_info( + document_item, + user_id=( + document_item.get("user_id") + if scope == SOURCE_SCOPE_PERSONAL + else None + ), + group_id=( + document_item.get("group_id") + if scope == SOURCE_SCOPE_GROUP + else None + ), + public_workspace_id=( + document_item.get("public_workspace_id") + if scope == SOURCE_SCOPE_PUBLIC + else None + ), + ) + if blob_container and blob_path: + storage_locator = { + "container": str(blob_container), + "blob_path": str(blob_path), + } + except Exception: + storage_locator = None + return { "document_id": document_id, "display_name": display_name, @@ -238,6 +445,7 @@ def _build_authorized_manifest_entry(document_id, user_id, document_context): "public_workspace_id": public_workspace_id, "conversation_id": conversation_id, "source_version": source_version, + "storage_locator": storage_locator, "authorization_status": AUTHORIZATION_STATUS_AUTHORIZED, } @@ -257,6 +465,7 @@ def resolve_authorized_source_manifest( conversation_id=None, active_group_ids=None, active_public_workspace_ids=None, + doc_scope="all", context_resolver=None, ): """Resolve each unique requested ID once into an ordered, authorized manifest.""" @@ -307,6 +516,9 @@ def resolve_authorized_source_manifest( active_public_workspace_ids ) normalized_conversation_id = str(conversation_id or "").strip() or None + normalized_doc_scope = str(doc_scope or "all").strip().lower() + if normalized_doc_scope not in {"all", "personal", "group", "public"}: + raise ValueError("doc_scope must be all, personal, group, or public") resolved_contexts = None if context_resolver is None: @@ -314,7 +526,7 @@ def resolve_authorized_source_manifest( resolved_contexts = _default_document_context_batch_resolver( document_ids=unique_document_ids, user_id=normalized_user_id, - doc_scope="all", + doc_scope=normalized_doc_scope, active_group_ids=normalized_active_group_ids, active_public_workspace_id=normalized_public_workspace_ids, conversation_id=normalized_conversation_id, @@ -338,13 +550,20 @@ def resolve_authorized_source_manifest( document_context = context_resolver( document_id=document_id, user_id=normalized_user_id, - doc_scope="all", + doc_scope=normalized_doc_scope, active_group_ids=normalized_active_group_ids, active_public_workspace_id=normalized_public_workspace_ids, conversation_id=normalized_conversation_id, ) except Exception: resolution_error_count += 1 + if ( + isinstance(document_context, dict) + and normalized_doc_scope != "all" + and str(document_context.get("scope") or "").strip().lower() + != normalized_doc_scope + ): + document_context = None manifest.append( _build_authorized_manifest_entry( document_id, @@ -681,4 +900,296 @@ def serialize_evidence_envelope(envelope): ) if len(serialized_envelope.encode("utf-8")) > EVIDENCE_ENVELOPE_MAX_BYTES: raise ValueError("Evidence envelope exceeds its serialized size bound") - return serialized_envelope \ No newline at end of file + return serialized_envelope + + +def build_narrative_evidence_envelopes( + narrative_sources, + search_results, + selection_mode, +): + """Normalize bounded narrative search results into one envelope per source.""" + normalized_selection_mode = normalize_selection_mode( + selection_mode, + default=SELECTION_MODE_RELEVANCE, + ) + results_by_document_id = {} + for raw_result in list(search_results or []): + result = raw_result if isinstance(raw_result, dict) else {} + document_id = str(result.get("document_id") or "").strip() + if not document_id: + continue + results_by_document_id.setdefault(document_id, []).append(result) + + envelopes = [] + for source in list(narrative_sources or []): + source = source if isinstance(source, dict) else {} + document_id = str(source.get("document_id") or "").strip() + if not document_id: + continue + source_results = results_by_document_id.get(document_id, []) + evidence = [] + citations = [] + for result in source_results: + evidence.append({ + "chunk_text": result.get("chunk_text"), + "page_number": result.get("page_number"), + "chunk_sequence": result.get("chunk_sequence"), + "score": result.get("score"), + }) + citations.append({ + "citation_id": result.get("id") or result.get("chunk_id"), + "page_number": result.get("page_number"), + "chunk_sequence": result.get("chunk_sequence"), + }) + + result_count = len(source_results) + envelopes.append(build_evidence_envelope( + document_id=document_id, + source_kind=SOURCE_KIND_NARRATIVE, + engine=EVIDENCE_ENGINE_HYBRID_SEARCH, + status=( + EVIDENCE_STATUS_COMPLETED + if result_count + else EVIDENCE_STATUS_PARTIAL + ), + summary=( + f"Retrieved {result_count} bounded narrative excerpt(s)." + if result_count + else "No relevant narrative excerpts were returned." + ), + evidence=evidence, + citations=citations, + coverage={ + "selection_mode": normalized_selection_mode, + "terminal": True, + "result_count": result_count, + }, + error=( + None + if result_count + else "Narrative retrieval returned no relevant excerpts." + ), + )) + return envelopes + + +def execute_tabular_evidence_sources( + tabular_sources, + execute_source, + selection_mode, + execute=True, +): + """Execute the existing tabular runner once per source and require terminal coverage.""" + normalized_selection_mode = normalize_selection_mode( + selection_mode, + default=SELECTION_MODE_RELEVANCE, + ) + if execute and not callable(execute_source): + raise TypeError("execute_source must be callable") + + envelopes = [] + completed_count = 0 + failed_count = 0 + skipped_count = 0 + for raw_source in list(tabular_sources or []): + source = raw_source if isinstance(raw_source, dict) else {} + document_id = str(source.get("document_id") or "").strip() + if not document_id: + continue + + if not execute: + skipped_count += 1 + envelopes.append(build_evidence_envelope( + document_id=document_id, + source_kind=SOURCE_KIND_TABULAR, + engine=EVIDENCE_ENGINE_TABULAR_TOOLS, + status=EVIDENCE_STATUS_SKIPPED, + summary="Tabular processing was not needed for this narrative-only request.", + coverage={ + "selection_mode": normalized_selection_mode, + "terminal": True, + "reason": "narrative_only_request", + }, + )) + continue + + try: + raw_result = execute_source(source) + result = raw_result if isinstance(raw_result, dict) else {} + summary = str(result.get("summary") or "").strip() + if not summary: + raise ValueError("Tabular execution returned no bounded summary") + completed_count += 1 + envelopes.append(build_evidence_envelope( + document_id=document_id, + source_kind=SOURCE_KIND_TABULAR, + engine=EVIDENCE_ENGINE_TABULAR_TOOLS, + status=EVIDENCE_STATUS_COMPLETED, + summary=summary, + evidence=result.get("evidence"), + citations=result.get("citations"), + generated_artifacts=result.get("generated_artifacts"), + coverage={ + "selection_mode": normalized_selection_mode, + "terminal": True, + **dict(result.get("coverage") or {}), + }, + )) + except Exception: + failed_count += 1 + envelopes.append(build_evidence_envelope( + document_id=document_id, + source_kind=SOURCE_KIND_TABULAR, + engine=EVIDENCE_ENGINE_TABULAR_TOOLS, + status=EVIDENCE_STATUS_FAILED, + summary="Tabular evidence could not be completed for this source.", + coverage={ + "selection_mode": normalized_selection_mode, + "terminal": True, + }, + error="Tabular evidence could not be completed.", + )) + + log_event( + "[MixedSourceChatSearch] Tabular source execution reached terminal coverage.", + extra={ + "selection_mode": normalized_selection_mode, + "tabular_candidate_count": len(list(tabular_sources or [])), + "tabular_completed_count": completed_count, + "tabular_failed_count": failed_count, + "tabular_skipped_count": skipped_count, + }, + level=logging.INFO, + ) + return envelopes + + +def build_mixed_source_evidence_handoff( + manifest, + evidence_envelopes, + selection_mode, +): + """Build one bounded synthesis handoff from Phase 1 evidence envelopes.""" + normalized_selection_mode = normalize_selection_mode( + selection_mode, + default=SELECTION_MODE_RELEVANCE, + ) + manifest_entries = [entry for entry in list(manifest or []) if isinstance(entry, dict)] + all_envelopes = [ + envelope + for envelope in list(evidence_envelopes or []) + if isinstance(envelope, dict) + ] + envelopes = all_envelopes[:MIXED_SOURCE_HANDOFF_MAX_ENVELOPES] + envelope_by_document_id = { + str(envelope.get("document_id") or "").strip(): envelope + for envelope in all_envelopes + if str(envelope.get("document_id") or "").strip() + } + evidence_omitted_count = max(0, len(all_envelopes) - len(envelopes)) + + source_coverage = [] + failed_count = 0 + partial_count = 0 + skipped_count = 0 + completed_count = 0 + for source_index, entry in enumerate(manifest_entries, start=1): + document_id = str(entry.get("document_id") or "").strip() + envelope = envelope_by_document_id.get(document_id) + if entry.get("authorization_status") != AUTHORIZATION_STATUS_AUTHORIZED: + status = EVIDENCE_STATUS_FAILED + source_label = f"Unavailable selected source {source_index}" + elif entry.get("source_kind") == SOURCE_KIND_UNSUPPORTED: + status = EVIDENCE_STATUS_FAILED + source_label = str(entry.get("display_name") or f"Unsupported source {source_index}") + elif envelope: + status = envelope.get("status") or EVIDENCE_STATUS_FAILED + source_label = str(entry.get("display_name") or f"Source {source_index}") + else: + status = EVIDENCE_STATUS_FAILED + source_label = str(entry.get("display_name") or f"Source {source_index}") + + if status == EVIDENCE_STATUS_COMPLETED: + completed_count += 1 + elif status == EVIDENCE_STATUS_PARTIAL: + partial_count += 1 + elif status == EVIDENCE_STATUS_SKIPPED: + skipped_count += 1 + else: + failed_count += 1 + source_coverage.append({ + "source": _truncate_utf8(source_label, 128), + "source_kind": entry.get("source_kind"), + "status": status, + }) + + coverage = { + "selection_mode": normalized_selection_mode, + "requested_source_count": len(manifest_entries), + "completed_source_count": completed_count, + "partial_source_count": partial_count, + "failed_source_count": failed_count, + "skipped_source_count": skipped_count, + "partial_coverage": bool( + failed_count or partial_count or evidence_omitted_count + ), + "evidence_omitted_count": evidence_omitted_count, + "sources": source_coverage, + } + payload = { + "coverage": coverage, + "evidence_envelopes": envelopes, + } + serialized_payload = json.dumps( + payload, + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + ) + if len(serialized_payload.encode("utf-8")) > MIXED_SOURCE_HANDOFF_MAX_BYTES: + payload["evidence_envelopes"] = [ + { + "document_id": envelope.get("document_id"), + "source_kind": envelope.get("source_kind"), + "engine": envelope.get("engine"), + "status": envelope.get("status"), + "summary": _truncate_utf8(envelope.get("summary"), 512), + "coverage": { + "selection_mode": (envelope.get("coverage") or {}).get("selection_mode"), + "terminal": bool((envelope.get("coverage") or {}).get("terminal")), + "result_count": (envelope.get("coverage") or {}).get("result_count"), + "tool_call_count": (envelope.get("coverage") or {}).get("tool_call_count"), + }, + } + for envelope in envelopes + ] + payload["coverage"]["handoff_compacted"] = True + payload["coverage"]["partial_coverage"] = True + payload["coverage"]["evidence_compacted_count"] = len(envelopes) + serialized_payload = json.dumps( + payload, + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + ) + if len(serialized_payload.encode("utf-8")) > MIXED_SOURCE_HANDOFF_MAX_BYTES: + raise ValueError("Mixed-source evidence handoff exceeds its size bound") + + partial_coverage_instruction = ( + "State clearly that source coverage was partial and identify unavailable authorized source labels." + if coverage["partial_coverage"] + else "Do not claim that selected sources were omitted." + ) + return { + "role": "system", + "content": ( + "Use the mixed-source evidence handoff below with the other bounded narrative excerpts " + "and computed tabular results. Synthesize one answer. Preserve narrative source citations " + "and tabular tool citations; do not convert computed table facts into unsupported narrative claims. " + "When selection_mode is selected, current selected-source evidence supersedes prior document " + "grounding; do not use prior source claims to fill missing current coverage. " + f"{partial_coverage_instruction}\n\n{serialized_payload}" + ), + "mixed_source_coverage": coverage, + } \ No newline at end of file diff --git a/application/single_app/functions_search_service.py b/application/single_app/functions_search_service.py index becbe4502..17e116947 100644 --- a/application/single_app/functions_search_service.py +++ b/application/single_app/functions_search_service.py @@ -45,6 +45,9 @@ SUMMARY_MAX_WINDOW_SIZE = 50 CHAT_UPLOAD_CHUNK_WORD_SIZE = 400 CHAT_UPLOAD_CHUNK_WORD_OVERLAP = 40 +MIXED_SOURCE_TABULAR_CANDIDATE_TOP_N = 36 +MIXED_SOURCE_TABULAR_CANDIDATE_LIMIT = 6 +MIXED_SOURCE_TABULAR_EXTENSIONS = frozenset({".csv", ".xls", ".xlsx", ".xlsm"}) def _coerce_positive_int(value, default_value, min_value=1, max_value=None): @@ -553,6 +556,7 @@ def build_search_request( active_group_ids=None, active_public_workspace_id=None, enable_file_sharing=True, + include_all_public_workspaces=False, ): normalized_query = str(query or "").strip() if not normalized_query: @@ -592,7 +596,11 @@ def build_search_request( active_public_workspace_id=active_public_workspace_id, ) if resolved_public_workspace_ids and normalized_scope in ("all", "public"): - search_request["active_public_workspace_id"] = resolved_public_workspace_ids[0] + search_request["active_public_workspace_id"] = ( + resolved_public_workspace_ids + if include_all_public_workspaces + else resolved_public_workspace_ids[0] + ) return search_request @@ -608,6 +616,7 @@ def search_documents( active_group_ids=None, active_public_workspace_id=None, enable_file_sharing=True, + include_all_public_workspaces=False, ): search_request = build_search_request( query=query, @@ -620,6 +629,7 @@ def search_documents( active_group_ids=active_group_ids, active_public_workspace_id=active_public_workspace_id, enable_file_sharing=enable_file_sharing, + include_all_public_workspaces=include_all_public_workspaces, ) results = hybrid_search(**search_request) or [] unique_document_ids = { @@ -642,6 +652,70 @@ def search_documents( } +def search_relevant_tabular_candidates( + query, + user_id, + doc_scope="all", + document_ids=None, + tags_filter=None, + active_group_ids=None, + active_public_workspace_id=None, + max_candidates=MIXED_SOURCE_TABULAR_CANDIDATE_LIMIT, +): + """Find a bounded set of authorized table candidates from indexed schema chunks.""" + normalized_limit = _coerce_positive_int( + max_candidates, + MIXED_SOURCE_TABULAR_CANDIDATE_LIMIT, + min_value=1, + max_value=MIXED_SOURCE_TABULAR_CANDIDATE_LIMIT, + ) + candidate_query = ( + f"{str(query or '').strip()}\n" + "Relevant spreadsheet, workbook, worksheet, CSV, table schema, columns, and data fields." + ).strip() + search_result = search_documents( + query=candidate_query, + user_id=user_id, + top_n=MIXED_SOURCE_TABULAR_CANDIDATE_TOP_N, + doc_scope=doc_scope, + document_ids=document_ids, + tags_filter=tags_filter, + active_group_ids=active_group_ids, + active_public_workspace_id=active_public_workspace_id, + include_all_public_workspaces=True, + ) + + candidate_document_ids = [] + seen_document_ids = set() + for result in search_result.get("results") or []: + file_name = str(result.get("file_name") or "").strip() + if os.path.splitext(file_name)[1].lower() not in MIXED_SOURCE_TABULAR_EXTENSIONS: + continue + document_id = str(result.get("document_id") or "").strip() + if not document_id or document_id in seen_document_ids: + continue + seen_document_ids.add(document_id) + candidate_document_ids.append(document_id) + if len(candidate_document_ids) >= normalized_limit: + break + + log_event( + "[MixedSourceChatSearch] Completed bounded authorized tabular candidate search.", + extra={ + "candidate_search_result_count": search_result.get("result_count", 0), + "tabular_candidate_count": len(candidate_document_ids), + "candidate_limit": normalized_limit, + }, + level=logging.INFO, + ) + return { + "document_ids": candidate_document_ids, + "candidate_count": len(candidate_document_ids), + "search_result_count": search_result.get("result_count", 0), + "query": search_result.get("query"), + } + + def _derive_window_size(chunks, window_unit, window_size=None, window_percent=None): if not chunks: return 0 diff --git a/application/single_app/functions_settings.py b/application/single_app/functions_settings.py index 6db9782cf..4c6462f32 100644 --- a/application/single_app/functions_settings.py +++ b/application/single_app/functions_settings.py @@ -215,6 +215,19 @@ def is_mixed_source_manifest_enabled(settings): return bool((settings or {}).get('enable_mixed_source_manifest', False)) +def is_mixed_source_chat_search_enabled(settings): + """Return whether Phase 2 mixed-source Chat and Search behavior is enabled.""" + return bool((settings or {}).get('enable_mixed_source_chat_search', False)) + + +def is_mixed_source_relevance_candidates_enabled(settings): + """Return whether Phase 2 relevance-mode table candidates are enabled.""" + return bool( + (settings or {}).get('enable_mixed_source_chat_search', False) + and (settings or {}).get('enable_mixed_source_relevance_candidates', False) + ) + + CHAT_FILE_UPLOAD_APP_ROLE = "ChatFileUploadUser" WORKFLOW_USER_APP_ROLE = "WorkflowUser" DOCUMENT_INTELLIGENCE_PDF_IMAGE_EXTRACTION_MODES = {"read", "layout", "auto"} @@ -765,6 +778,8 @@ def get_settings(use_cosmos=False, include_source=False): 'enable_tabular_processing_plugin': False, 'enable_multi_agent_orchestration': False, 'enable_mixed_source_manifest': False, + 'enable_mixed_source_chat_search': False, + 'enable_mixed_source_relevance_candidates': False, 'max_rounds_per_agent': 1, 'workflow_max_auto_invoke_attempts': 60, 'enable_semantic_kernel': False, diff --git a/application/single_app/functions_tabular_analysis.py b/application/single_app/functions_tabular_analysis.py index ae6421c4e..3f8eedcf1 100644 --- a/application/single_app/functions_tabular_analysis.py +++ b/application/single_app/functions_tabular_analysis.py @@ -12,6 +12,7 @@ def _load_chat_helper(helper_name): # Import lazily because route_backend_chats imports functions_workflow_runner during app startup. from route_backend_chats import ( + _execute_mixed_source_tabular_evidence, augment_tabular_invocations_with_related_document_evidence, build_tabular_computed_results_system_message, build_tabular_related_document_evidence_summary, @@ -21,6 +22,7 @@ def _load_chat_helper(helper_name): ) helpers = { + 'execute_mixed_source_tabular_evidence': _execute_mixed_source_tabular_evidence, 'augment_tabular_invocations_with_related_document_evidence': augment_tabular_invocations_with_related_document_evidence, 'build_tabular_computed_results_system_message': build_tabular_computed_results_system_message, 'build_tabular_related_document_evidence_summary': build_tabular_related_document_evidence_summary, @@ -31,6 +33,10 @@ def _load_chat_helper(helper_name): return helpers[helper_name] +def execute_mixed_source_tabular_evidence(*args, **kwargs): + return _load_chat_helper('execute_mixed_source_tabular_evidence')(*args, **kwargs) + + def augment_tabular_invocations_with_related_document_evidence(*args, **kwargs): return _load_chat_helper('augment_tabular_invocations_with_related_document_evidence')(*args, **kwargs) diff --git a/application/single_app/functions_workflow_runner.py b/application/single_app/functions_workflow_runner.py index 4087561fa..3afaa1a09 100644 --- a/application/single_app/functions_workflow_runner.py +++ b/application/single_app/functions_workflow_runner.py @@ -78,7 +78,12 @@ build_agent_citation_artifact_documents, make_json_serializable, ) -from functions_mixed_source_orchestration import resolve_authorized_source_manifest +from functions_mixed_source_orchestration import ( + build_mixed_source_evidence_handoff, + build_narrative_evidence_envelopes, + partition_source_manifest, + resolve_authorized_source_manifest, +) from model_endpoint_clients import ( MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI, ) @@ -86,12 +91,17 @@ from functions_notifications import create_workflow_priority_notification from functions_personal_workflows import save_personal_workflow_run, save_personal_workflow_run_item from functions_public_workspaces import get_user_visible_public_workspace_ids_from_settings -from functions_search_service import resolve_document_context, search_documents +from functions_search_service import ( + resolve_document_context, + search_documents, + search_relevant_tabular_candidates, +) from functions_search import normalize_search_id_list, normalize_search_scope, normalize_search_top_n from functions_simplechat_operations import upload_generated_analysis_artifact_for_current_user from functions_settings import ( get_settings, get_user_settings, + is_mixed_source_chat_search_enabled, is_mixed_source_manifest_enabled, is_tabular_processing_enabled, normalize_model_endpoints, @@ -3515,6 +3525,8 @@ def _create_assistant_message(conversation, workflow, result, trigger_source, ru 'workspace_type': workspace_type, 'group_id': group_id or None, 'token_usage': result.get('token_usage'), + 'generated_analysis_artifacts': list(result.get('generated_analysis_artifacts') or []), + 'generated_tabular_outputs': list(result.get('generated_tabular_outputs') or []), 'source_review': source_review_metadata, 'workflow': { 'workflow_id': workflow.get('id'), @@ -3527,6 +3539,7 @@ def _create_assistant_message(conversation, workflow, result, trigger_source, ru 'model_binding_summary': workflow.get('model_binding_summary') or {}, 'document_action': document_action, 'document_search': result.get('document_search') or {}, + 'mixed_source_coverage': result.get('mixed_source_coverage') or {}, 'analyze': workflow.get('analyze') or {}, 'analysis_coverage': result.get('analysis_coverage') or {}, }, @@ -4023,20 +4036,86 @@ def _format_workflow_search_results(results): def _build_workflow_search_prompt(task_prompt, search_context): task_prompt = str(task_prompt or '').strip() retrieved_content = str((search_context or {}).get('retrieved_content') or '').strip() - if not retrieved_content: + evidence_messages = [ + str((message or {}).get('content') or '').strip() + for message in list((search_context or {}).get('evidence_messages') or []) + if str((message or {}).get('content') or '').strip() + ] + if not retrieved_content and not evidence_messages: return task_prompt - return ( + prompt_sections = [ '[Workflow document search context]\n' - 'Use the retrieved document excerpts below as grounding for the workflow task. ' - 'When the excerpts are insufficient, say what is missing instead of guessing.\n\n' - f'{retrieved_content}\n\n' - '[Workflow task]\n' - f'{task_prompt}' - ).strip() + 'Use the bounded native-engine evidence below as grounding for the workflow task. ' + 'When the evidence is insufficient or source coverage is partial, say what is missing instead of guessing.' + ] + if retrieved_content: + prompt_sections.append( + f'[Narrative excerpts]\n{retrieved_content}' + ) + if evidence_messages: + prompt_sections.append( + '[Computed and coverage evidence]\n' + + '\n\n'.join(evidence_messages) + ) + prompt_sections.append(f'[Workflow task]\n{task_prompt}') + return '\n\n'.join(prompt_sections).strip() -def _prepare_workflow_search_context(workflow, action_config, settings, thought_tracker=None, run_id=None): +@contextmanager +def _workflow_mixed_source_execution_context(user_id, conversation_id, manifest): + """Bind tabular tools to scope IDs from a freshly authorized manifest.""" + group_ids = [] + public_workspace_ids = [] + for source in list(manifest or []): + if source.get('authorization_status') != 'authorized': + continue + group_id = str(source.get('group_id') or '').strip() + public_workspace_id = str(source.get('public_workspace_id') or '').strip() + if group_id and group_id not in group_ids: + group_ids.append(group_id) + if public_workspace_id and public_workspace_id not in public_workspace_ids: + public_workspace_ids.append(public_workspace_id) + + with _ensure_execution_context(user_id): + previous_conversation_id = getattr(g, 'conversation_id', None) if hasattr(g, 'conversation_id') else None + previous_authorized_chat_context = getattr(g, 'authorized_chat_context', None) if hasattr(g, 'authorized_chat_context') else None + g.conversation_id = conversation_id + g.authorized_chat_context = { + 'user_id': user_id, + 'conversation_id': conversation_id, + 'active_group_ids': group_ids, + 'active_group_id': group_ids[0] if len(group_ids) == 1 else None, + 'active_public_workspace_ids': public_workspace_ids, + 'active_public_workspace_id': ( + public_workspace_ids[0] + if len(public_workspace_ids) == 1 + else None + ), + 'fact_memory_scope_id': group_ids[0] if len(group_ids) == 1 else user_id, + 'fact_memory_scope_type': 'group' if len(group_ids) == 1 else 'user', + } + try: + yield + finally: + if previous_conversation_id is None and hasattr(g, 'conversation_id'): + delattr(g, 'conversation_id') + else: + g.conversation_id = previous_conversation_id + if previous_authorized_chat_context is None and hasattr(g, 'authorized_chat_context'): + delattr(g, 'authorized_chat_context') + else: + g.authorized_chat_context = previous_authorized_chat_context + + +def _prepare_workflow_search_context( + workflow, + action_config, + settings, + conversation_id='', + thought_tracker=None, + run_id=None, +): if not _is_document_search_workflow(action_config): return {'workflow': workflow, 'citations': [], 'result_count': 0, 'document_count': 0, 'query': None} @@ -4049,20 +4128,163 @@ def _prepare_workflow_search_context(workflow, action_config, settings, thought_ if not query: return {'workflow': workflow, 'citations': [], 'result_count': 0, 'document_count': 0, 'query': None} - search_top_n = normalize_search_top_n(max(12, len(document_ids) * 3 if document_ids else 12)) - search_result = search_documents( - query=query, - user_id=str(workflow.get('user_id') or '').strip(), - top_n=search_top_n, - doc_scope=resolved_action.get('doc_scope') or 'all', - document_ids=document_ids, - active_group_ids=resolved_action.get('active_group_ids'), - active_public_workspace_id=resolved_action.get('active_public_workspace_id'), + user_id = str(workflow.get('user_id') or '').strip() + manifest_doc_scope = resolved_action.get('doc_scope') or 'all' + manifest_group_ids = list(resolved_action.get('active_group_ids') or []) + manifest_public_workspace_ids = list( + resolved_action.get('active_public_workspace_id') or [] ) + workflow_group_id = _get_workflow_group_id(workflow) + if workflow_group_id: + assert_group_role( + user_id, + workflow_group_id, + allowed_roles=("Owner", "Admin", "DocumentManager", "User"), + ) + manifest_doc_scope = 'group' + manifest_group_ids = [workflow_group_id] + manifest_public_workspace_ids = [] + + scoped_action = dict(resolved_action) + scoped_action['doc_scope'] = manifest_doc_scope + scoped_action['active_group_ids'] = manifest_group_ids + scoped_action['active_public_workspace_id'] = manifest_public_workspace_ids + + if not is_mixed_source_chat_search_enabled(settings): + search_top_n = normalize_search_top_n(max(12, len(document_ids) * 3 if document_ids else 12)) + search_result = search_documents( + query=query, + user_id=user_id, + top_n=search_top_n, + doc_scope=manifest_doc_scope, + document_ids=document_ids, + active_group_ids=manifest_group_ids, + active_public_workspace_id=manifest_public_workspace_ids, + ) + retrieved_content, citations = _format_workflow_search_results(search_result.get('results') or []) + prepared_workflow = _apply_runtime_document_action_config(workflow, scoped_action) + prepared_workflow['task_prompt'] = _build_workflow_search_prompt(workflow.get('task_prompt', ''), { + 'retrieved_content': retrieved_content, + }) + + if thought_tracker and run_id: + _add_workflow_activity_thought( + thought_tracker, + prepared_workflow, + run_id, + step_type='document', + content='Searched selected workflow documents', + detail=( + f"results={search_result.get('result_count', 0)} | " + f"documents={search_result.get('document_count', 0)}" + ), + activity_key=f'search:{run_id}:documents', + kind='document_search', + title='Document search', + status='completed', + ) + + return { + 'workflow': prepared_workflow, + 'citations': citations, + 'agent_citations': [], + 'generated_tabular_outputs': [], + 'coverage': {}, + 'result_count': search_result.get('result_count', 0), + 'document_count': search_result.get('document_count', 0), + 'query': search_result.get('query'), + } + + manifest = resolve_authorized_source_manifest( + document_ids, + user_id=user_id, + selection_mode='selected', + conversation_id=conversation_id, + active_group_ids=manifest_group_ids, + active_public_workspace_ids=manifest_public_workspace_ids, + doc_scope=manifest_doc_scope, + ) + partitions = partition_source_manifest(manifest) + narrative_sources = list(partitions.get('narrative_sources') or []) + tabular_sources = list(partitions.get('tabular_sources') or []) + narrative_document_ids = [ + str(source.get('document_id') or '').strip() + for source in narrative_sources + if str(source.get('document_id') or '').strip() + ] + authorized_document_ids = narrative_document_ids + [ + str(source.get('document_id') or '').strip() + for source in tabular_sources + if str(source.get('document_id') or '').strip() + ] + + manifest_group_ids = [] + manifest_public_workspace_ids = [] + for source in narrative_sources + tabular_sources: + group_id = str(source.get('group_id') or '').strip() + public_workspace_id = str(source.get('public_workspace_id') or '').strip() + if group_id and group_id not in manifest_group_ids: + manifest_group_ids.append(group_id) + if public_workspace_id and public_workspace_id not in manifest_public_workspace_ids: + manifest_public_workspace_ids.append(public_workspace_id) + + search_top_n = normalize_search_top_n( + max(12, len(narrative_document_ids) * 3 if narrative_document_ids else 12) + ) + search_result = { + 'results': [], + 'result_count': 0, + 'document_count': 0, + 'query': query, + } + if narrative_document_ids: + search_result = search_documents( + query=query, + user_id=user_id, + top_n=search_top_n, + doc_scope=manifest_doc_scope, + document_ids=narrative_document_ids, + active_group_ids=manifest_group_ids, + active_public_workspace_id=manifest_public_workspace_ids, + include_all_public_workspaces=True, + ) retrieved_content, citations = _format_workflow_search_results(search_result.get('results') or []) - prepared_workflow = _apply_runtime_document_action_config(workflow, resolved_action) + evidence_envelopes = build_narrative_evidence_envelopes( + narrative_sources, + search_result.get('results') or [], + 'selected', + ) + + from functions_tabular_analysis import execute_mixed_source_tabular_evidence + + with _workflow_mixed_source_execution_context(user_id, conversation_id, manifest): + tabular_result = execute_mixed_source_tabular_evidence( + tabular_sources=tabular_sources, + selection_mode='selected', + has_narrative_sources=bool(narrative_sources), + user_question=query, + user_id=user_id, + conversation_id=conversation_id, + gpt_model=_resolve_tabular_document_action_model_name(workflow, settings), + settings=settings, + thought_tracker=thought_tracker, + ) + evidence_envelopes.extend(tabular_result.get('evidence_envelopes') or []) + mixed_source_handoff = build_mixed_source_evidence_handoff( + manifest, + evidence_envelopes, + 'selected', + ) + + prepared_action = dict(resolved_action) + prepared_action['document_ids'] = authorized_document_ids + prepared_action['doc_scope'] = manifest_doc_scope + prepared_action['active_group_ids'] = manifest_group_ids + prepared_action['active_public_workspace_id'] = manifest_public_workspace_ids + prepared_workflow = _apply_runtime_document_action_config(workflow, prepared_action) prepared_workflow['task_prompt'] = _build_workflow_search_prompt(workflow.get('task_prompt', ''), { - 'retrieved_content': retrieved_content, + 'retrieved_content': '', + 'evidence_messages': [mixed_source_handoff], }) if thought_tracker and run_id: @@ -4074,7 +4296,8 @@ def _prepare_workflow_search_context(workflow, action_config, settings, thought_ content='Searched selected workflow documents', detail=( f"results={search_result.get('result_count', 0)} | " - f"documents={search_result.get('document_count', 0)}" + f"narrative_documents={len(narrative_sources)} | " + f"tabular_documents={len(tabular_sources)}" ), activity_key=f'search:{run_id}:documents', kind='document_search', @@ -4085,12 +4308,48 @@ def _prepare_workflow_search_context(workflow, action_config, settings, thought_ return { 'workflow': prepared_workflow, 'citations': citations, + 'agent_citations': list(tabular_result.get('agent_citations') or []), + 'generated_tabular_outputs': list(tabular_result.get('generated_outputs') or []), + 'coverage': mixed_source_handoff.get('mixed_source_coverage') or {}, 'result_count': search_result.get('result_count', 0), - 'document_count': search_result.get('document_count', 0), + 'document_count': len(authorized_document_ids), 'query': search_result.get('query'), } +def _attach_workflow_search_context(execution_result, workflow_search_context): + """Merge mixed Search evidence into either a workflow model or agent result.""" + execution_result = execution_result if isinstance(execution_result, dict) else {} + search_context = workflow_search_context if isinstance(workflow_search_context, dict) else {} + if not search_context: + return execution_result + + existing_agent_citations = list(execution_result.get('agent_citations') or []) + existing_agent_citations.extend(list(search_context.get('agent_citations') or [])) + existing_outputs = list(execution_result.get('generated_tabular_outputs') or []) + existing_outputs.extend(list(search_context.get('generated_tabular_outputs') or [])) + hybrid_citations = list(search_context.get('citations') or []) + coverage = search_context.get('coverage') if isinstance(search_context.get('coverage'), dict) else {} + execution_result.update({ + 'hybrid_citations': hybrid_citations, + 'agent_citations': existing_agent_citations, + 'generated_tabular_outputs': existing_outputs, + 'mixed_source_coverage': coverage, + 'augmented': bool( + hybrid_citations + or existing_agent_citations + or coverage.get('requested_source_count') + ), + 'document_search': { + 'query': search_context.get('query'), + 'result_count': search_context.get('result_count', 0), + 'document_count': search_context.get('document_count', 0), + 'partial_coverage': bool(coverage.get('partial_coverage')), + }, + }) + return execution_result + + def _apply_runtime_document_action_config(workflow, action_config): prepared_workflow = dict(workflow or {}) prepared_workflow['document_action'] = dict(action_config or {}) @@ -5713,6 +5972,7 @@ def run_personal_workflow(workflow, trigger_source='manual', user_roles=None, ac execution_workflow, document_action, settings, + conversation_id=conversation.get('id'), thought_tracker=thought_tracker, run_id=run_id, ) @@ -5751,16 +6011,10 @@ def run_personal_workflow(workflow, trigger_source='manual', user_roles=None, ac thought_tracker=thought_tracker, url_access_context=url_access_context, ) - if workflow_search_context: - execution_result.update({ - 'hybrid_citations': workflow_search_context.get('citations') or [], - 'augmented': bool(workflow_search_context.get('citations')), - 'document_search': { - 'query': workflow_search_context.get('query'), - 'result_count': workflow_search_context.get('result_count', 0), - 'document_count': workflow_search_context.get('document_count', 0), - }, - }) + execution_result = _attach_workflow_search_context( + execution_result, + workflow_search_context, + ) else: execution_result = _execute_model_workflow( execution_workflow, @@ -5769,16 +6023,10 @@ def run_personal_workflow(workflow, trigger_source='manual', user_roles=None, ac thought_tracker=thought_tracker, url_access_context=url_access_context, ) - if workflow_search_context: - execution_result.update({ - 'hybrid_citations': workflow_search_context.get('citations') or [], - 'augmented': bool(workflow_search_context.get('citations')), - 'document_search': { - 'query': workflow_search_context.get('query'), - 'result_count': workflow_search_context.get('result_count', 0), - 'document_count': workflow_search_context.get('document_count', 0), - }, - }) + execution_result = _attach_workflow_search_context( + execution_result, + workflow_search_context, + ) execution_result = _attach_workflow_url_access_result(execution_result, url_access_context) assistant_doc = _create_assistant_message( diff --git a/application/single_app/route_backend_chats.py b/application/single_app/route_backend_chats.py index 03c967be5..83f60a89c 100644 --- a/application/single_app/route_backend_chats.py +++ b/application/single_app/route_backend_chats.py @@ -37,7 +37,16 @@ build_model_endpoint_sync_chat_client, build_semantic_kernel_chat_service_for_model, ) -from functions_mixed_source_orchestration import resolve_authorized_source_manifest +from functions_mixed_source_orchestration import ( + build_mixed_source_evidence_handoff, + build_narrative_evidence_envelopes, + build_tabular_file_contexts_from_manifest, + execute_tabular_evidence_sources, + normalize_document_context_request, + partition_source_manifest, + resolve_authorized_source_manifest, + should_run_tabular_evidence, +) import builtins import asyncio, types import ast @@ -60,6 +69,7 @@ from flask import Response, copy_current_request_context, g, has_request_context, stream_with_context from functions_authentication import * from functions_search import * +from functions_search_service import search_relevant_tabular_candidates from functions_service_health import ( SEMANTIC_SEARCH_QUOTA_WARNING_TYPE, SemanticSearchQuotaExceededError, @@ -181,6 +191,7 @@ } ASSIGNED_KNOWLEDGE_CONTEXT_TOP_N = 12 ASSIGNED_KNOWLEDGE_CONTEXT_EXCERPT_MAX_CHARS = 1800 +MIXED_SOURCE_CHAT_RELEVANCE_SOURCE_LIMIT = 48 FOUNDRY_SELECTED_AGENT_TYPES = {'aifoundry', 'new_foundry', 'foundry_workflow'} FOUNDRY_AGENT_PLUGIN_NAMES = { 'aifoundry': 'azure_ai_foundry', @@ -328,6 +339,180 @@ def _maybe_resolve_chat_source_manifest( return [] +def _normalize_chat_document_context_contract( + settings, + data, + selected_document_ids, + hybrid_search_enabled, +): + """Normalize Phase 2 context intent while preserving flag-off compatibility.""" + normalized_document_ids = _normalize_conversation_task_document_ids( + selected_document_ids + ) + if not is_mixed_source_chat_search_enabled(settings): + return { + 'selection_mode': 'selected' if normalized_document_ids else 'relevance', + 'selected_document_ids': normalized_document_ids, + 'document_context_requested': bool(hybrid_search_enabled), + 'hybrid_search': bool(hybrid_search_enabled), + 'explicit_selection': False, + } + + return normalize_document_context_request( + selection_mode=data.get('selection_mode'), + selected_document_ids=normalized_document_ids, + document_context_requested=data.get('document_context_requested'), + hybrid_search=hybrid_search_enabled, + ) + + +def _resolve_chat_mixed_source_manifest( + settings, + user_id, + conversation_id, + document_ids, + selection_mode, + active_group_ids=None, + active_public_workspace_ids=None, +): + """Resolve a fresh Phase 1 manifest for Phase 2 Chat evidence preparation.""" + if not is_mixed_source_chat_search_enabled(settings): + return [] + normalized_document_ids = _normalize_conversation_task_document_ids(document_ids) + if not normalized_document_ids: + return [] + return resolve_authorized_source_manifest( + normalized_document_ids, + user_id=user_id, + selection_mode=selection_mode, + conversation_id=conversation_id, + active_group_ids=active_group_ids, + active_public_workspace_ids=active_public_workspace_ids, + ) + + +def _get_manifest_partition_document_ids(manifest_partitions, partition_name): + partitions = manifest_partitions if isinstance(manifest_partitions, dict) else {} + return [ + str(source.get('document_id') or '').strip() + for source in partitions.get(partition_name) or [] + if str(source.get('document_id') or '').strip() + ] + + +def _resolve_chat_mixed_source_partition( + settings, + user_id, + conversation_id, + document_ids, + selection_mode, + active_group_ids=None, + active_public_workspace_ids=None, +): + """Resolve and partition one current authorization-safe source manifest.""" + manifest = _resolve_chat_mixed_source_manifest( + settings, + user_id, + conversation_id, + document_ids, + selection_mode, + active_group_ids=active_group_ids, + active_public_workspace_ids=active_public_workspace_ids, + ) + partitions = partition_source_manifest(manifest) + return { + 'manifest': manifest, + 'partitions': partitions, + 'narrative_document_ids': _get_manifest_partition_document_ids( + partitions, + 'narrative_sources', + ), + 'tabular_sources': list(partitions.get('tabular_sources') or []), + } + + +def _resolve_chat_mixed_source_relevance_context( + *, + settings, + user_id, + conversation_id, + query, + search_results, + document_scope, + candidate_document_ids=None, + tags_filter=None, + active_group_ids=None, + active_public_workspace_ids=None, +): + """Add bounded schema candidates and reauthorize all relevance-derived sources.""" + if not is_mixed_source_chat_search_enabled(settings): + return { + 'manifest': [], + 'partitions': {}, + 'narrative_document_ids': [], + 'tabular_sources': [], + 'search_results': list(search_results or []), + 'tabular_candidate_count': 0, + } + + relevance_candidates_enabled = is_mixed_source_relevance_candidates_enabled(settings) + candidate_result = { + 'document_ids': [], + 'candidate_count': 0, + } + if relevance_candidates_enabled: + candidate_result = search_relevant_tabular_candidates( + query=query, + user_id=user_id, + doc_scope=document_scope, + document_ids=candidate_document_ids, + tags_filter=tags_filter, + active_group_ids=active_group_ids, + active_public_workspace_id=active_public_workspace_ids, + ) + relevance_document_ids = [] + seen_document_ids = set() + for result in list(search_results or []): + document_id = str((result or {}).get('document_id') or '').strip() + if document_id and document_id not in seen_document_ids: + seen_document_ids.add(document_id) + relevance_document_ids.append(document_id) + if len(relevance_document_ids) >= ( + MIXED_SOURCE_CHAT_RELEVANCE_SOURCE_LIMIT - 6 + ): + break + for document_id in candidate_result.get('document_ids') or []: + normalized_document_id = str(document_id or '').strip() + if normalized_document_id and normalized_document_id not in seen_document_ids: + seen_document_ids.add(normalized_document_id) + relevance_document_ids.append(normalized_document_id) + if len(relevance_document_ids) >= MIXED_SOURCE_CHAT_RELEVANCE_SOURCE_LIMIT: + break + + resolved_context = _resolve_chat_mixed_source_partition( + settings, + user_id, + conversation_id, + relevance_document_ids, + 'relevance', + active_group_ids=active_group_ids, + active_public_workspace_ids=active_public_workspace_ids, + ) + narrative_document_id_set = set( + resolved_context.get('narrative_document_ids') or [] + ) + resolved_context['search_results'] = [ + result + for result in list(search_results or []) + if str((result or {}).get('document_id') or '').strip() + in narrative_document_id_set + ] + resolved_context['tabular_candidate_count'] = int( + candidate_result.get('candidate_count') or 0 + ) + return resolved_context + + def _source_review_metadata_used(source_review_result): if not isinstance(source_review_result, dict): return False @@ -9508,6 +9693,17 @@ async def run_tabular_sk_analysis(user_question, tabular_filenames, user_id, file_source_hint = file_context.get('source_hint', source_hint) file_group_id = file_context.get('group_id') file_public_workspace_id = file_context.get('public_workspace_id') + storage_locator = file_context.get('storage_locator') + if isinstance(storage_locator, dict): + locator_container = str(storage_locator.get('container') or '').strip() + locator_blob_path = str(storage_locator.get('blob_path') or '').strip() + if locator_container and locator_blob_path: + tabular_plugin.remember_resolved_blob_location( + file_source_hint, + fname, + locator_container, + locator_blob_path, + ) schema_source_context = {'source': file_source_hint} if file_group_id: schema_source_context['group_id'] = file_group_id @@ -10460,12 +10656,8 @@ def build_system_prompt(force_tool_use=False, tool_error_messages=None, log_event(f"[Tabular SK Analysis] Error: {e}", level=logging.WARNING, exceptionTraceback=True) return None -def collect_tabular_sk_citations(user_id, conversation_id): - """Collect plugin invocations from the tabular SK analysis and convert to citation format.""" - from semantic_kernel_plugins.plugin_invocation_logger import get_plugin_logger - - plugin_logger = get_plugin_logger() - plugin_invocations = plugin_logger.get_invocations_for_conversation(user_id, conversation_id) +def _build_tabular_sk_citations_from_invocations(plugin_invocations): + """Convert selected tabular invocations to the existing tool-citation shape.""" plugin_invocations = filter_tabular_citation_invocations(plugin_invocations) if not plugin_invocations: @@ -10510,10 +10702,254 @@ def collect_tabular_sk_citations(user_id, conversation_id): } citations.append(citation) - log_event(f"[Tabular SK Citations] Collected {len(citations)} tool execution citations", level=logging.INFO) return citations +def collect_tabular_sk_citations(user_id, conversation_id): + """Collect plugin invocations from the tabular SK analysis and convert to citation format.""" + plugin_logger = get_plugin_logger() + plugin_invocations = plugin_logger.get_invocations_for_conversation(user_id, conversation_id) + citations = _build_tabular_sk_citations_from_invocations(plugin_invocations) + log_event( + f"[Tabular SK Citations] Collected {len(citations)} tool execution citations", + level=logging.INFO, + ) + return citations + + +def _execute_mixed_source_tabular_evidence( + *, + tabular_sources, + selection_mode, + has_narrative_sources, + user_question, + user_id, + conversation_id, + gpt_model, + settings, + thought_tracker=None, + live_thought_callback=None, + model_context=None, +): + """Run the existing tabular engine once per manifest source with terminal coverage.""" + source_contexts = build_tabular_file_contexts_from_manifest(tabular_sources) + if has_request_context(): + authorized_context = dict(getattr(g, 'authorized_chat_context', {}) or {}) + authorized_blob_locations = [] + for file_context in source_contexts: + storage_locator = file_context.get('storage_locator') + if not isinstance(storage_locator, dict): + continue + locator_container = str(storage_locator.get('container') or '').strip() + locator_blob_path = str(storage_locator.get('blob_path') or '').strip() + if locator_container and locator_blob_path: + authorized_blob_locations.append([locator_container, locator_blob_path]) + authorized_context['authorized_blob_locations'] = authorized_blob_locations + g.authorized_chat_context = authorized_context + context_by_document_id = { + context['document_id']: context + for context in source_contexts + } + system_messages = [] + agent_citations = [] + generated_outputs = [] + all_invocations = [] + if not is_tabular_processing_enabled(settings): + def fail_unavailable_tabular_source(source): + del source + raise RuntimeError('Tabular processing is unavailable') + + return { + 'evidence_envelopes': execute_tabular_evidence_sources( + tabular_sources, + fail_unavailable_tabular_source, + selection_mode, + ), + 'system_messages': [], + 'agent_citations': [], + 'generated_outputs': [], + 'invocations': [], + 'executed': False, + } + + execute_tabular = should_run_tabular_evidence( + user_question, + has_narrative_sources=has_narrative_sources, + ) + plugin_logger = get_plugin_logger() + + def publish_post_processing_thought(thought_payload): + payload = thought_payload if isinstance(thought_payload, dict) else {} + if thought_tracker is not None: + thought_tracker.add_thought( + payload.get('step_type', 'tabular_analysis'), + payload.get('content', ''), + detail=payload.get('detail'), + activity=payload.get('activity'), + ) + if callable(live_thought_callback): + live_payload = dict(payload) + if thought_tracker is not None: + live_payload['message_id'] = getattr(thought_tracker, 'message_id', None) + live_payload['step_index'] = thought_tracker.current_index - 1 + live_thought_callback(live_payload) + + def execute_source(source): + document_id = str(source.get('document_id') or '').strip() + file_context = context_by_document_id.get(document_id) + if not file_context: + raise ValueError('Authorized tabular source context is unavailable') + + baseline_invocation_count = len( + plugin_logger.get_invocations_for_conversation( + user_id, + conversation_id, + limit=1000, + ) + ) + execution_mode = get_tabular_execution_mode(user_question) + tabular_analysis, streamed_tool_thoughts = asyncio.run( + run_tabular_analysis_with_thought_tracking( + user_question=user_question, + tabular_filenames={file_context['file_name']}, + tabular_file_contexts=[file_context], + user_id=user_id, + conversation_id=conversation_id, + gpt_model=gpt_model, + settings=settings, + source_hint=file_context.get('source_hint', 'workspace'), + group_id=file_context.get('group_id'), + public_workspace_id=file_context.get('public_workspace_id'), + execution_mode=execution_mode, + thought_tracker=thought_tracker, + live_thought_callback=live_thought_callback, + model_context=model_context, + ) + ) + if not str(tabular_analysis or '').strip(): + raise ValueError('Tabular execution returned no computed results') + + invocations_after = plugin_logger.get_invocations_for_conversation( + user_id, + conversation_id, + limit=1000, + ) + source_invocations = get_new_plugin_invocations( + invocations_after, + baseline_invocation_count, + ) + if execution_mode == 'schema_summary': + successful_source_invocations = [ + invocation + for invocation in source_invocations + if getattr(invocation, 'function_name', '') == 'describe_tabular_file' + and not get_tabular_invocation_error_message(invocation) + ] + else: + successful_source_invocations, _ = split_tabular_analysis_invocations( + source_invocations + ) + if not successful_source_invocations: + raise ValueError('Tabular execution returned no successful native tool calls') + all_invocations.extend(source_invocations) + related_document_summary = '' + related_document_stats = augment_tabular_invocations_with_related_document_evidence( + source_invocations, + user_question, + user_id, + conversation_id=conversation_id, + ) + if related_document_stats.get('augmented_row_count'): + related_document_summary = build_tabular_related_document_evidence_summary( + source_invocations, + ) + + if thought_tracker is not None and not streamed_tool_thoughts: + for thought_content, thought_detail in get_tabular_tool_thought_payloads( + source_invocations + ): + thought_tracker.add_thought( + 'tabular_analysis', + thought_content, + thought_detail, + ) + if thought_tracker is not None: + for thought_content, thought_detail in get_tabular_status_thought_payloads( + source_invocations, + analysis_succeeded=True, + ): + thought_tracker.add_thought( + 'tabular_analysis', + thought_content, + thought_detail, + ) + + generated_output = asyncio.run(maybe_create_tabular_generated_output( + user_question=user_question, + invocations=source_invocations, + gpt_model=gpt_model, + settings=settings, + conversation_id=conversation_id, + thought_callback=publish_post_processing_thought, + user_id=user_id, + model_context=model_context, + )) + if generated_output: + generated_outputs.append(generated_output) + + source_citations = _build_tabular_sk_citations_from_invocations( + source_invocations + ) + agent_citations.extend(source_citations) + chart_citations = build_tabular_inline_chart_citations( + user_question, + source_invocations, + ) + agent_citations.extend(chart_citations) + system_messages.append({ + 'role': 'system', + 'content': build_tabular_computed_results_system_message( + 'an authorized selected tabular source', + str(tabular_analysis).strip(), + related_document_evidence_summary=related_document_summary, + ), + }) + if generated_output: + system_messages.append({ + 'role': 'system', + 'content': _build_tabular_generated_output_system_message(generated_output), + }) + + return { + 'summary': str(tabular_analysis).strip(), + 'evidence': [ + get_tabular_invocation_compact_payload(invocation, max_rows=5) + for invocation in source_invocations + ], + 'citations': source_citations, + 'generated_artifacts': [generated_output] if generated_output else [], + 'coverage': { + 'tool_call_count': len(source_invocations), + 'execution_mode': execution_mode, + }, + } + + evidence_envelopes = execute_tabular_evidence_sources( + tabular_sources, + execute_source, + selection_mode, + execute=execute_tabular, + ) + return { + 'evidence_envelopes': evidence_envelopes, + 'system_messages': system_messages, + 'agent_citations': agent_citations, + 'generated_outputs': generated_outputs, + 'invocations': all_invocations, + 'executed': execute_tabular, + } + + def is_tabular_filename(filename): """Return True when the filename has a supported tabular extension.""" if not filename or not isinstance(filename, str): @@ -13206,6 +13642,38 @@ def result_requires_message_reload(result: Any) -> bool: deep_research_enabled = deep_research_enabled.lower() == 'true' if isinstance(image_gen_enabled, str): image_gen_enabled = image_gen_enabled.lower() == 'true' + try: + document_context_contract = _normalize_chat_document_context_contract( + settings, + data, + selected_document_ids, + hybrid_search_enabled, + ) + except ValueError as contract_error: + return jsonify({'error': str(contract_error)}), 400 + selected_document_ids = list( + document_context_contract.get('selected_document_ids') or [] + ) + requested_selected_document_ids = list(selected_document_ids) + selected_document_id = ( + selected_document_ids[0] + if len(selected_document_ids) == 1 + else None + ) + selection_mode = document_context_contract.get('selection_mode') + document_context_requested = bool( + document_context_contract.get('document_context_requested') + ) + request_has_explicit_document_selection = bool( + document_context_contract.get('explicit_selection') + ) + request_document_context_enabled = bool( + hybrid_search_enabled + or ( + is_mixed_source_chat_search_enabled(settings) + and document_context_requested + ) + ) user_workspace_context_requested = data.get('user_workspace_context_enabled') if isinstance(user_workspace_context_requested, str): user_workspace_context_requested = user_workspace_context_requested.lower() == 'true' @@ -13273,7 +13741,7 @@ def result_requires_message_reload(result: Any) -> bool: assigned_knowledge_deep_research_urls = [] if assigned_knowledge_filters: assigned_knowledge_user_context_active = ( - user_workspace_context_requested + (user_workspace_context_requested or document_context_requested) and _assigned_knowledge_allows_user_workspace_context(assigned_knowledge_filters) and _assigned_knowledge_allows_document_action( assigned_knowledge_filters, @@ -13333,6 +13801,15 @@ def result_requires_message_reload(result: Any) -> bool: g.assigned_knowledge_context = assigned_knowledge_filters g.assigned_knowledge_user_context_active = assigned_knowledge_user_context_active + mixed_source_explicit_selection = bool( + is_mixed_source_chat_search_enabled(settings) + and request_has_explicit_document_selection + and ( + not assigned_knowledge_filters + or assigned_knowledge_user_context_active + ) + ) + explicit_external_retrieval_requested = _is_explicit_external_retrieval_requested( web_search_enabled=web_search_enabled, url_access_enabled=url_access_enabled, @@ -13503,11 +13980,14 @@ def result_requires_message_reload(result: Any) -> bool: _set_authorized_chat_request_context(user_id, conversation_id, scope_context) auto_linked_chat_upload_document_ids = [] - auto_merge_chat_upload_workspace_context = _should_auto_merge_chat_upload_workspace_context( - explicit_external_retrieval_requested, - hybrid_search_enabled, - assigned_knowledge_filters=assigned_knowledge_filters, - assigned_knowledge_user_context_active=assigned_knowledge_user_context_active, + auto_merge_chat_upload_workspace_context = ( + not mixed_source_explicit_selection + and _should_auto_merge_chat_upload_workspace_context( + explicit_external_retrieval_requested, + hybrid_search_enabled, + assigned_knowledge_filters=assigned_knowledge_filters, + assigned_knowledge_user_context_active=assigned_knowledge_user_context_active, + ) ) if auto_merge_chat_upload_workspace_context: chat_upload_context = _resolve_chat_upload_workspace_context( @@ -13566,12 +14046,76 @@ def result_requires_message_reload(result: Any) -> bool: selected_document_id = effective_selected_document_id document_scope = effective_document_scope - _maybe_resolve_chat_source_manifest( - settings, - user_id, - conversation_id, - effective_selected_document_ids, - scope_context, + mixed_source_manifest = [] + mixed_source_partitions = {} + mixed_source_narrative_document_ids = [] + mixed_source_tabular_sources = [] + mixed_source_evidence_envelopes = [] + if mixed_source_explicit_selection: + try: + mixed_source_manifest = _resolve_chat_mixed_source_manifest( + settings, + user_id, + conversation_id, + effective_selected_document_ids, + 'selected', + active_group_ids=effective_active_group_ids, + active_public_workspace_ids=effective_active_public_workspace_ids, + ) + except ValueError as manifest_error: + return jsonify({'error': str(manifest_error)}), 400 + mixed_source_partitions = partition_source_manifest( + mixed_source_manifest + ) + mixed_source_narrative_document_ids = _get_manifest_partition_document_ids( + mixed_source_partitions, + 'narrative_sources', + ) + mixed_source_tabular_sources = list( + mixed_source_partitions.get('tabular_sources') or [] + ) + authorized_selected_document_ids = [ + str(source.get('document_id') or '').strip() + for source in ( + list(mixed_source_partitions.get('narrative_sources') or []) + + mixed_source_tabular_sources + ) + if str(source.get('document_id') or '').strip() + ] + effective_selected_document_ids = authorized_selected_document_ids + effective_selected_document_id = ( + authorized_selected_document_ids[0] + if len(authorized_selected_document_ids) == 1 + else None + ) + log_event( + '[MixedSourceChatSearch] Activated explicit selected-source context.', + extra={ + 'selection_mode': 'selected', + 'requested_source_count': len(mixed_source_manifest), + 'authorized_source_count': len(authorized_selected_document_ids), + 'narrative_source_count': len(mixed_source_narrative_document_ids), + 'tabular_source_count': len(mixed_source_tabular_sources), + 'omitted_source_count': len( + mixed_source_partitions.get('unresolved_sources') or [] + ), + }, + level=logging.INFO, + ) + else: + _maybe_resolve_chat_source_manifest( + settings, + user_id, + conversation_id, + effective_selected_document_ids, + scope_context, + ) + request_document_context_enabled = bool( + hybrid_search_enabled + or ( + is_mixed_source_chat_search_enabled(settings) + and document_context_requested + ) ) # Clear plugin invocations at start of message processing to ensure @@ -13671,13 +14215,13 @@ def result_requires_message_reload(result: Any) -> bool: # Button states and selections user_metadata['button_states'] = { 'image_generation': image_gen_enabled, - 'document_search': hybrid_search_enabled, + 'document_search': request_document_context_enabled, 'web_search': bool(web_search_enabled), 'url_access': bool(url_access_enabled), 'deep_research': bool(deep_research_enabled) } user_metadata['capability_usage'] = _build_capability_usage_metadata( - workspace_search_enabled=hybrid_search_enabled, + workspace_search_enabled=request_document_context_enabled, document_action_type=DOCUMENT_ACTION_TYPE_NONE, document_scope=effective_document_scope, selected_document_ids=effective_selected_document_ids, @@ -13690,12 +14234,18 @@ def result_requires_message_reload(result: Any) -> bool: ) # Document search scope and selections - if hybrid_search_enabled: + if request_document_context_enabled: user_metadata['workspace_search'] = { 'search_enabled': True, + 'selection_mode': selection_mode, + 'document_context_requested': document_context_requested, + 'hybrid_search_preference': bool(hybrid_search_enabled), 'document_scope': effective_document_scope, 'selected_document_id': effective_selected_document_id, 'selected_document_ids': effective_selected_document_ids, + 'requested_document_ids': requested_selected_document_ids, + 'active_group_ids': effective_active_group_ids, + 'active_public_workspace_ids': effective_active_public_workspace_ids, 'tags': tags_filter, 'classification': classifications_to_send } @@ -13884,7 +14434,7 @@ def result_requires_message_reload(result: Any) -> bool: conversation_id=conversation_id, message_type='user_message', message_length=len(user_message) if user_message else 0, - has_document_search=hybrid_search_enabled, + has_document_search=request_document_context_enabled, has_image_generation=image_gen_enabled, document_scope=document_scope, chat_context=actual_chat_type, @@ -14029,7 +14579,11 @@ def result_requires_message_reload(result: Any) -> bool: except Exception as ex: debug_print(f"[Content Safety] Unexpected error: {ex}") - if not original_hybrid_search_enabled and not explicit_external_retrieval_requested: + if ( + not original_hybrid_search_enabled + and not explicit_external_retrieval_requested + and not mixed_source_explicit_selection + ): prior_grounded_document_refs = _normalize_prior_grounded_document_refs(conversation_item) if prior_grounded_document_refs: thought_tracker.add_thought( @@ -14143,13 +14697,64 @@ def result_requires_message_reload(result: Any) -> bool: 'history_context', 'No prior grounded documents were available; using conversation history only' ) + if ( + is_mixed_source_chat_search_enabled(settings) + and history_grounded_search_used + ): + history_context = _resolve_chat_mixed_source_partition( + settings, + user_id, + conversation_id, + effective_selected_document_ids, + 'history', + active_group_ids=effective_active_group_ids, + active_public_workspace_ids=effective_active_public_workspace_ids, + ) + mixed_source_manifest = history_context.get('manifest') or [] + mixed_source_partitions = history_context.get('partitions') or {} + mixed_source_narrative_document_ids = list( + history_context.get('narrative_document_ids') or [] + ) + mixed_source_tabular_sources = list( + history_context.get('tabular_sources') or [] + ) + effective_selected_document_ids = ( + mixed_source_narrative_document_ids + + _get_manifest_partition_document_ids( + mixed_source_partitions, + 'tabular_sources', + ) + ) + effective_selected_document_id = ( + effective_selected_document_ids[0] + if len(effective_selected_document_ids) == 1 + else None + ) # region 4 - Augmentation # --------------------------------------------------------------------- # 4) Augmentation (Search, etc.) - Run *before* final history prep # --------------------------------------------------------------------- # Hybrid Search - if hybrid_search_enabled or history_grounded_search_used: + mixed_source_document_context_active = bool( + (hybrid_search_enabled or history_grounded_search_used) + if not is_mixed_source_chat_search_enabled(settings) + else ( + document_context_requested + or hybrid_search_enabled + or history_grounded_search_used + ) + ) + combined_documents = [] + mixed_source_narrative_search_active = bool( + mixed_source_document_context_active + and ( + not is_mixed_source_chat_search_enabled(settings) + or not mixed_source_manifest + or mixed_source_narrative_document_ids + ) + ) + if mixed_source_narrative_search_active: # Optional: Summarize recent history *for search* (uses its own limit) if hybrid_search_enabled and enable_summarize_content_history_for_search: @@ -14264,8 +14869,14 @@ def result_requires_message_reload(result: Any) -> bool: ): search_args["active_public_workspace_id"] = effective_active_public_workspace_id - if effective_selected_document_ids: - search_args["document_ids"] = effective_selected_document_ids + search_document_ids = ( + mixed_source_narrative_document_ids + if is_mixed_source_chat_search_enabled(settings) + and mixed_source_manifest + else effective_selected_document_ids + ) + if search_document_ids: + search_args["document_ids"] = search_document_ids elif effective_selected_document_id: search_args["document_id"] = effective_selected_document_id if auto_linked_chat_upload_document_ids: @@ -14300,6 +14911,40 @@ def result_requires_message_reload(result: Any) -> bool: else: # Public scope now automatically searches all visible public workspaces search_results = hybrid_search(**search_args) # Assuming hybrid_search handles None document_id + + if ( + is_mixed_source_chat_search_enabled(settings) + and not mixed_source_explicit_selection + and not history_grounded_search_used + ): + relevance_context = _resolve_chat_mixed_source_relevance_context( + settings=settings, + user_id=user_id, + conversation_id=conversation_id, + query=search_query, + search_results=search_results, + document_scope=effective_document_scope, + candidate_document_ids=( + assigned_knowledge_filters.get('document_ids') + if assigned_knowledge_filters + and assigned_knowledge_filters.get('has_workspace_knowledge') + else None + ), + tags_filter=tags_filter, + active_group_ids=effective_active_group_ids, + active_public_workspace_ids=effective_active_public_workspace_ids, + ) + mixed_source_manifest = relevance_context.get('manifest') or [] + mixed_source_partitions = relevance_context.get('partitions') or {} + mixed_source_narrative_document_ids = list( + relevance_context.get('narrative_document_ids') or [] + ) + mixed_source_tabular_sources = list( + relevance_context.get('tabular_sources') or [] + ) + search_results = list( + relevance_context.get('search_results') or [] + ) except SemanticSearchQuotaExceededError as e: debug_print(f"Semantic search quota exceeded during hybrid search: {e}") return jsonify({ @@ -14314,7 +14959,6 @@ def result_requires_message_reload(result: Any) -> bool: 'error': 'There was an issue with the embedding process. Please check with an admin on embedding configuration.' }), 500 - combined_documents = [] if search_results: unique_doc_names = set(doc.get('file_name', 'Unknown') for doc in search_results) thought_tracker.add_thought('search', f"Found {len(search_results)} results from {len(unique_doc_names)} documents") @@ -14375,11 +15019,12 @@ def result_requires_message_reload(result: Any) -> bool: # Construct system prompt for search results system_prompt_search = build_search_augmentation_system_prompt(retrieved_content) # Add this to a temporary list, don't save to DB yet - system_messages_for_augmentation.append({ - 'role': 'system', - 'content': system_prompt_search, - 'documents': combined_documents # Keep track of docs used - }) + if not is_mixed_source_chat_search_enabled(settings): + system_messages_for_augmentation.append({ + 'role': 'system', + 'content': system_prompt_search, + 'documents': combined_documents # Keep track of docs used + }) # Loop through each source document/chunk used for this message for source_doc in combined_documents: @@ -14595,7 +15240,13 @@ def result_requires_message_reload(result: Any) -> bool: # Update message-level chat_type based on actual document usage for this message # This must happen after document search is completed so search_results is populated message_chat_type = None - if (hybrid_search_enabled or history_grounded_search_used) and search_results and len(search_results) > 0: + mixed_source_has_authorized_evidence_sources = bool( + mixed_source_narrative_document_ids + or mixed_source_tabular_sources + ) + if mixed_source_document_context_active and ( + search_results or mixed_source_has_authorized_evidence_sources + ): # Documents were actually used for this message if effective_document_scope == 'group': message_chat_type = 'group' @@ -14653,8 +15304,10 @@ def result_requires_message_reload(result: Any) -> bool: source_review_used = _source_review_metadata_used(source_review_result) user_metadata['capability_usage'] = _build_capability_usage_metadata( - workspace_search_enabled=hybrid_search_enabled or history_grounded_search_used, - workspace_search_used=bool(search_results), + workspace_search_enabled=mixed_source_document_context_active, + workspace_search_used=bool( + search_results or mixed_source_has_authorized_evidence_sources + ), workspace_search_result_count=len(search_results or []), document_action_type=DOCUMENT_ACTION_TYPE_NONE, document_scope=effective_document_scope, @@ -14863,7 +15516,11 @@ def result_requires_message_reload(result: Any) -> bool: workspace_tabular_file_contexts = [] workspace_tabular_files = set() - if (hybrid_search_enabled or history_grounded_search_used) and is_tabular_processing_enabled(settings): + if ( + not is_mixed_source_chat_search_enabled(settings) + and (hybrid_search_enabled or history_grounded_search_used) + and is_tabular_processing_enabled(settings) + ): workspace_tabular_file_contexts = collect_workspace_tabular_file_contexts( combined_documents=combined_documents, selected_document_ids=effective_selected_document_ids, @@ -14887,7 +15544,88 @@ def record_tabular_post_processing_thought(thought_payload): activity=thought_payload.get('activity'), ) - if (hybrid_search_enabled or history_grounded_search_used) and workspace_tabular_files and is_tabular_processing_enabled(settings): + if ( + is_mixed_source_chat_search_enabled(settings) + and mixed_source_document_context_active + and mixed_source_manifest + ): + effective_mixed_source_selection_mode = ( + 'selected' + if mixed_source_explicit_selection + else 'history' + if history_grounded_search_used + else 'relevance' + ) + mixed_source_evidence_envelopes.extend( + build_narrative_evidence_envelopes( + mixed_source_partitions.get('narrative_sources') or [], + search_results, + effective_mixed_source_selection_mode, + ) + ) + mixed_source_tabular_result = _execute_mixed_source_tabular_evidence( + tabular_sources=mixed_source_tabular_sources, + selection_mode=effective_mixed_source_selection_mode, + has_narrative_sources=bool(mixed_source_narrative_document_ids), + user_question=user_message, + user_id=user_id, + conversation_id=conversation_id, + gpt_model=gpt_model, + settings=settings, + thought_tracker=thought_tracker, + model_context=tabular_model_context, + ) + mixed_source_evidence_envelopes.extend( + mixed_source_tabular_result.get('evidence_envelopes') or [] + ) + agent_citations_list.extend( + mixed_source_tabular_result.get('agent_citations') or [] + ) + generated_tabular_outputs_list.extend( + mixed_source_tabular_result.get('generated_outputs') or [] + ) + generated_analysis_artifacts_list.extend( + mixed_source_tabular_result.get('generated_outputs') or [] + ) + mixed_source_handoff = build_mixed_source_evidence_handoff( + mixed_source_manifest, + mixed_source_evidence_envelopes, + effective_mixed_source_selection_mode, + ) + system_messages_for_augmentation.append(mixed_source_handoff) + mixed_source_coverage = mixed_source_handoff.get( + 'mixed_source_coverage', + {}, + ) + user_metadata['mixed_source_coverage'] = mixed_source_coverage + user_message_doc['metadata'] = user_metadata + cosmos_messages_container.upsert_item(user_message_doc) + log_event( + '[MixedSourceChatSearch] Prepared bounded mixed-source synthesis evidence.', + extra={ + 'selection_mode': effective_mixed_source_selection_mode, + 'narrative_result_count': len(search_results or []), + 'tabular_candidate_count': len(mixed_source_tabular_sources), + 'tabular_completed_count': sum( + 1 + for envelope in mixed_source_evidence_envelopes + if envelope.get('source_kind') == 'tabular' + and envelope.get('status') == 'completed' + ), + 'mixed_synthesis_count': 1, + 'partial_coverage': bool( + mixed_source_coverage.get('partial_coverage') + ), + }, + level=logging.INFO, + ) + + if ( + not is_mixed_source_chat_search_enabled(settings) + and (hybrid_search_enabled or history_grounded_search_used) + and workspace_tabular_files + and is_tabular_processing_enabled(settings) + ): tabular_source_hint = determine_tabular_source_hint( effective_document_scope, active_group_id=effective_active_group_id, @@ -15150,7 +15888,10 @@ def record_tabular_post_processing_thought(thought_payload): gpt_model=gpt_model, user_message_id=user_message_id, fallback_user_message=user_message, - include_assistant_citation_context=not explicit_external_retrieval_requested, + include_assistant_citation_context=( + not explicit_external_retrieval_requested + and not mixed_source_explicit_selection + ), ) summary_of_older = history_segments['summary_of_older'] chat_tabular_files = history_segments['chat_tabular_files'] @@ -15227,7 +15968,14 @@ def record_tabular_post_processing_thought(thought_payload): final_api_source_refs.extend(history_debug_info.get('history_message_source_refs', [])) # --- Mini SK analysis for tabular files uploaded directly to chat --- - if chat_tabular_files and is_tabular_processing_enabled(settings): + if ( + chat_tabular_files + and is_tabular_processing_enabled(settings) + and not ( + is_mixed_source_chat_search_enabled(settings) + and (mixed_source_explicit_selection or mixed_source_tabular_sources) + ) + ): chat_tabular_filenames_str = ", ".join(chat_tabular_files) chat_tabular_execution_mode = get_tabular_execution_mode(user_message) log_event( @@ -15390,6 +16138,10 @@ def record_tabular_post_processing_thought(thought_payload): original_hybrid_search_enabled, prior_grounded_document_refs, explicit_external_retrieval_requested, + current_document_context_requested=( + is_mixed_source_chat_search_enabled(settings) + and document_context_requested + ), ): history_grounding_message = build_history_grounding_system_message() insert_idx = 0 @@ -15828,10 +16580,12 @@ def invoke_foundry_agent(): 'user_id': user_id, 'message_id': user_message_id, 'chat_type': chat_type, - 'document_scope': document_scope, + 'document_scope': effective_document_scope, 'group_id': active_group_id if chat_type == 'group' else None, 'hybrid_search_enabled': hybrid_search_enabled, - 'selected_document_id': selected_document_id, + 'selection_mode': selection_mode, + 'document_context_requested': mixed_source_document_context_active, + 'selected_document_id': effective_selected_document_id, 'selected_document_ids': effective_selected_document_ids, 'active_group_ids': effective_active_group_ids, 'active_public_workspace_ids': effective_active_public_workspace_ids, @@ -16272,8 +17026,10 @@ def gpt_error(e): ) source_review_used = _source_review_metadata_used(source_review_result) assistant_capability_usage = _build_capability_usage_metadata( - workspace_search_enabled=hybrid_search_enabled or history_grounded_search_used, - workspace_search_used=bool(search_results), + workspace_search_enabled=mixed_source_document_context_active, + workspace_search_used=bool( + search_results or mixed_source_has_authorized_evidence_sources + ), workspace_search_result_count=len(hybrid_citations_list or []), document_action_type=DOCUMENT_ACTION_TYPE_NONE, document_scope=effective_document_scope, @@ -16430,7 +17186,7 @@ def gpt_error(e): document_scope=effective_document_scope, selected_document_id=effective_selected_document_id, model_deployment=actual_model_used, - hybrid_search_enabled=hybrid_search_enabled or history_grounded_search_used, + hybrid_search_enabled=mixed_source_document_context_active, image_gen_enabled=image_gen_enabled, selected_documents=combined_documents if 'combined_documents' in locals() else None, selected_agent=selected_agent_name, @@ -16867,6 +17623,39 @@ def stream_cancel_requested(): source_review_enabled = source_review_enabled.lower() == 'true' if isinstance(deep_research_enabled, str): deep_research_enabled = deep_research_enabled.lower() == 'true' + try: + document_context_contract = _normalize_chat_document_context_contract( + settings, + data, + selected_document_ids, + hybrid_search_enabled, + ) + except ValueError as contract_error: + yield f"data: {json.dumps({'error': str(contract_error)})}\n\n" + return + selected_document_ids = list( + document_context_contract.get('selected_document_ids') or [] + ) + requested_selected_document_ids = list(selected_document_ids) + selected_document_id = ( + selected_document_ids[0] + if len(selected_document_ids) == 1 + else None + ) + selection_mode = document_context_contract.get('selection_mode') + document_context_requested = bool( + document_context_contract.get('document_context_requested') + ) + request_has_explicit_document_selection = bool( + document_context_contract.get('explicit_selection') + ) + request_document_context_enabled = bool( + hybrid_search_enabled + or ( + is_mixed_source_chat_search_enabled(settings) + and document_context_requested + ) + ) user_workspace_context_requested = data.get('user_workspace_context_enabled') if isinstance(user_workspace_context_requested, str): user_workspace_context_requested = user_workspace_context_requested.lower() == 'true' @@ -16931,7 +17720,7 @@ def stream_cancel_requested(): assigned_knowledge_deep_research_urls = [] if assigned_knowledge_filters: assigned_knowledge_user_context_active = ( - user_workspace_context_requested + (user_workspace_context_requested or document_context_requested) and _assigned_knowledge_allows_user_workspace_context(assigned_knowledge_filters) and _assigned_knowledge_allows_document_action( assigned_knowledge_filters, @@ -16996,6 +17785,16 @@ def stream_cancel_requested(): f"public_workspaces={len(effective_active_public_workspace_ids)} | " f"tags={len(tags_filter)}" ) + mixed_source_explicit_selection = bool( + is_mixed_source_chat_search_enabled(settings) + and request_has_explicit_document_selection + and ( + not assigned_knowledge_filters + or assigned_knowledge_user_context_active + ) + ) + mixed_source_document_context_active = request_document_context_enabled + mixed_source_has_authorized_evidence_sources = False explicit_external_retrieval_requested = _is_explicit_external_retrieval_requested( web_search_enabled=web_search_enabled, url_access_enabled=url_access_enabled, @@ -17013,8 +17812,10 @@ def stream_cancel_requested(): def build_streaming_capability_usage(): source_review_was_used = _source_review_metadata_used(source_review_result) return _build_capability_usage_metadata( - workspace_search_enabled=hybrid_search_enabled or history_grounded_search_used, - workspace_search_used=bool(search_results), + workspace_search_enabled=mixed_source_document_context_active, + workspace_search_used=bool( + search_results or mixed_source_has_authorized_evidence_sources + ), workspace_search_result_count=len(hybrid_citations_list or []), document_action_type=DOCUMENT_ACTION_TYPE_NONE, document_scope=effective_document_scope, @@ -17181,11 +17982,14 @@ def build_streaming_capability_usage(): return auto_linked_chat_upload_document_ids = [] - auto_merge_chat_upload_workspace_context = _should_auto_merge_chat_upload_workspace_context( - explicit_external_retrieval_requested, - hybrid_search_enabled, - assigned_knowledge_filters=assigned_knowledge_filters, - assigned_knowledge_user_context_active=assigned_knowledge_user_context_active, + auto_merge_chat_upload_workspace_context = ( + not mixed_source_explicit_selection + and _should_auto_merge_chat_upload_workspace_context( + explicit_external_retrieval_requested, + hybrid_search_enabled, + assigned_knowledge_filters=assigned_knowledge_filters, + assigned_knowledge_user_context_active=assigned_knowledge_user_context_active, + ) ) if auto_merge_chat_upload_workspace_context: chat_upload_context = _resolve_chat_upload_workspace_context( @@ -17246,12 +18050,73 @@ def build_streaming_capability_usage(): selected_document_id = effective_selected_document_id document_scope = effective_document_scope - _maybe_resolve_chat_source_manifest( - settings, - user_id, - conversation_id, - effective_selected_document_ids, - scope_context, + mixed_source_manifest = [] + mixed_source_partitions = {} + mixed_source_narrative_document_ids = [] + mixed_source_tabular_sources = [] + mixed_source_evidence_envelopes = [] + if mixed_source_explicit_selection: + try: + explicit_context = _resolve_chat_mixed_source_partition( + settings, + user_id, + conversation_id, + effective_selected_document_ids, + 'selected', + active_group_ids=effective_active_group_ids, + active_public_workspace_ids=effective_active_public_workspace_ids, + ) + except ValueError as manifest_error: + yield f"data: {json.dumps({'error': str(manifest_error)})}\n\n" + return + mixed_source_manifest = explicit_context.get('manifest') or [] + mixed_source_partitions = explicit_context.get('partitions') or {} + mixed_source_narrative_document_ids = list( + explicit_context.get('narrative_document_ids') or [] + ) + mixed_source_tabular_sources = list( + explicit_context.get('tabular_sources') or [] + ) + effective_selected_document_ids = ( + mixed_source_narrative_document_ids + + _get_manifest_partition_document_ids( + mixed_source_partitions, + 'tabular_sources', + ) + ) + effective_selected_document_id = ( + effective_selected_document_ids[0] + if len(effective_selected_document_ids) == 1 + else None + ) + log_event( + '[MixedSourceChatSearch] Activated streaming explicit selected-source context.', + extra={ + 'selection_mode': 'selected', + 'requested_source_count': len(mixed_source_manifest), + 'authorized_source_count': len(effective_selected_document_ids), + 'narrative_source_count': len(mixed_source_narrative_document_ids), + 'tabular_source_count': len(mixed_source_tabular_sources), + 'omitted_source_count': len( + mixed_source_partitions.get('unresolved_sources') or [] + ), + }, + level=logging.INFO, + ) + else: + _maybe_resolve_chat_source_manifest( + settings, + user_id, + conversation_id, + effective_selected_document_ids, + scope_context, + ) + request_document_context_enabled = bool( + hybrid_search_enabled + or ( + is_mixed_source_chat_search_enabled(settings) + and document_context_requested + ) ) # Determine chat type @@ -17285,13 +18150,13 @@ def build_streaming_capability_usage(): user_metadata['button_states'] = { 'image_generation': False, - 'document_search': hybrid_search_enabled, + 'document_search': request_document_context_enabled, 'web_search': bool(web_search_enabled), 'url_access': bool(url_access_enabled), 'deep_research': bool(deep_research_enabled) } user_metadata['capability_usage'] = _build_capability_usage_metadata( - workspace_search_enabled=hybrid_search_enabled, + workspace_search_enabled=request_document_context_enabled, document_action_type=DOCUMENT_ACTION_TYPE_NONE, document_scope=effective_document_scope, selected_document_ids=effective_selected_document_ids, @@ -17304,12 +18169,16 @@ def build_streaming_capability_usage(): ) # Document search scope and selections - if hybrid_search_enabled: + if request_document_context_enabled: user_metadata['workspace_search'] = { 'search_enabled': True, + 'selection_mode': selection_mode, + 'document_context_requested': document_context_requested, + 'hybrid_search_preference': bool(hybrid_search_enabled), 'document_scope': effective_document_scope, 'selected_document_id': effective_selected_document_id, 'selected_document_ids': effective_selected_document_ids, + 'requested_document_ids': requested_selected_document_ids, 'active_group_ids': effective_active_group_ids, 'active_public_workspace_ids': effective_active_public_workspace_ids, 'classification': classifications_to_send @@ -17460,7 +18329,7 @@ def build_streaming_capability_usage(): conversation_id=conversation_id, message_type='user_message', message_length=len(user_message) if user_message else 0, - has_document_search=hybrid_search_enabled, + has_document_search=request_document_context_enabled, has_image_generation=False, document_scope=effective_document_scope, chat_context=actual_chat_type, @@ -17674,7 +18543,11 @@ def record_and_publish_streaming_thought(thought_payload): except Exception as ex: debug_print(f"[Content Safety - Streaming] Unexpected error: {ex}") - if not original_hybrid_search_enabled and not explicit_external_retrieval_requested: + if ( + not original_hybrid_search_enabled + and not explicit_external_retrieval_requested + and not mixed_source_explicit_selection + ): prior_grounded_document_refs = _normalize_prior_grounded_document_refs(conversation_item) if prior_grounded_document_refs: yield emit_thought( @@ -17789,9 +18662,60 @@ def record_and_publish_streaming_thought(thought_payload): 'No prior grounded documents were available; using conversation history only' ) + if ( + is_mixed_source_chat_search_enabled(settings) + and history_grounded_search_used + ): + history_context = _resolve_chat_mixed_source_partition( + settings, + user_id, + conversation_id, + effective_selected_document_ids, + 'history', + active_group_ids=effective_active_group_ids, + active_public_workspace_ids=effective_active_public_workspace_ids, + ) + mixed_source_manifest = history_context.get('manifest') or [] + mixed_source_partitions = history_context.get('partitions') or {} + mixed_source_narrative_document_ids = list( + history_context.get('narrative_document_ids') or [] + ) + mixed_source_tabular_sources = list( + history_context.get('tabular_sources') or [] + ) + effective_selected_document_ids = ( + mixed_source_narrative_document_ids + + _get_manifest_partition_document_ids( + mixed_source_partitions, + 'tabular_sources', + ) + ) + effective_selected_document_id = ( + effective_selected_document_ids[0] + if len(effective_selected_document_ids) == 1 + else None + ) + # Hybrid search (if enabled) combined_documents = [] - if hybrid_search_enabled or history_grounded_search_used: + mixed_source_document_context_active = bool( + (hybrid_search_enabled or history_grounded_search_used) + if not is_mixed_source_chat_search_enabled(settings) + else ( + document_context_requested + or hybrid_search_enabled + or history_grounded_search_used + ) + ) + mixed_source_narrative_search_active = bool( + mixed_source_document_context_active + and ( + not is_mixed_source_chat_search_enabled(settings) + or not mixed_source_manifest + or mixed_source_narrative_document_ids + ) + ) + if mixed_source_narrative_search_active: debug_print( "[Streaming] Starting hybrid search | " f"conversation_id={conversation_id} | doc_scope={effective_document_scope} | " @@ -17834,8 +18758,14 @@ def record_and_publish_streaming_thought(thought_payload): ): search_args['active_public_workspace_id'] = effective_active_public_workspace_id - if effective_selected_document_ids: - search_args['document_ids'] = effective_selected_document_ids + search_document_ids = ( + mixed_source_narrative_document_ids + if is_mixed_source_chat_search_enabled(settings) + and mixed_source_manifest + else effective_selected_document_ids + ) + if search_document_ids: + search_args['document_ids'] = search_document_ids elif effective_selected_document_id: search_args['document_id'] = effective_selected_document_id if auto_linked_chat_upload_document_ids: @@ -17865,6 +18795,40 @@ def record_and_publish_streaming_thought(thought_payload): search_results = assigned_search_results else: search_results = hybrid_search(**search_args) + + if ( + is_mixed_source_chat_search_enabled(settings) + and not mixed_source_explicit_selection + and not history_grounded_search_used + ): + relevance_context = _resolve_chat_mixed_source_relevance_context( + settings=settings, + user_id=user_id, + conversation_id=conversation_id, + query=search_query, + search_results=search_results, + document_scope=effective_document_scope, + candidate_document_ids=( + assigned_knowledge_filters.get('document_ids') + if assigned_knowledge_filters + and assigned_knowledge_filters.get('has_workspace_knowledge') + else None + ), + tags_filter=tags_filter, + active_group_ids=effective_active_group_ids, + active_public_workspace_ids=effective_active_public_workspace_ids, + ) + mixed_source_manifest = relevance_context.get('manifest') or [] + mixed_source_partitions = relevance_context.get('partitions') or {} + mixed_source_narrative_document_ids = list( + relevance_context.get('narrative_document_ids') or [] + ) + mixed_source_tabular_sources = list( + relevance_context.get('tabular_sources') or [] + ) + search_results = list( + relevance_context.get('search_results') or [] + ) debug_print( f"[Streaming] Hybrid search completed | results={len(search_results) if search_results else 0}" ) @@ -18070,11 +19034,12 @@ def record_and_publish_streaming_thought(thought_payload): retrieved_content = "\n\n".join(retrieved_texts) system_prompt_search = build_search_augmentation_system_prompt(retrieved_content) - system_messages_for_augmentation.append({ - 'role': 'system', - 'content': system_prompt_search, - 'documents': combined_documents - }) + if not is_mixed_source_chat_search_enabled(settings): + system_messages_for_augmentation.append({ + 'role': 'system', + 'content': system_prompt_search, + 'documents': combined_documents + }) hybrid_citations_list.sort(key=_build_hybrid_citation_sort_key, reverse=True) elif history_grounded_search_used: @@ -18103,7 +19068,11 @@ def record_and_publish_streaming_thought(thought_payload): workspace_tabular_file_contexts = [] workspace_tabular_files = set() - if (hybrid_search_enabled or history_grounded_search_used) and is_tabular_processing_enabled(settings): + if ( + not is_mixed_source_chat_search_enabled(settings) + and (hybrid_search_enabled or history_grounded_search_used) + and is_tabular_processing_enabled(settings) + ): workspace_tabular_file_contexts = collect_workspace_tabular_file_contexts( combined_documents=combined_documents, selected_document_ids=effective_selected_document_ids, @@ -18119,7 +19088,93 @@ def record_and_publish_streaming_thought(thought_payload): file_context['file_name'] for file_context in workspace_tabular_file_contexts } - if (hybrid_search_enabled or history_grounded_search_used) and workspace_tabular_files and is_tabular_processing_enabled(settings): + if ( + is_mixed_source_chat_search_enabled(settings) + and mixed_source_document_context_active + and mixed_source_manifest + ): + effective_mixed_source_selection_mode = ( + 'selected' + if mixed_source_explicit_selection + else 'history' + if history_grounded_search_used + else 'relevance' + ) + mixed_source_evidence_envelopes.extend( + build_narrative_evidence_envelopes( + mixed_source_partitions.get('narrative_sources') or [], + search_results, + effective_mixed_source_selection_mode, + ) + ) + mixed_source_tabular_result = _execute_mixed_source_tabular_evidence( + tabular_sources=mixed_source_tabular_sources, + selection_mode=effective_mixed_source_selection_mode, + has_narrative_sources=bool(mixed_source_narrative_document_ids), + user_question=user_message, + user_id=user_id, + conversation_id=conversation_id, + gpt_model=gpt_model, + settings=settings, + thought_tracker=thought_tracker, + live_thought_callback=publish_live_plugin_thought, + model_context=tabular_model_context, + ) + mixed_source_evidence_envelopes.extend( + mixed_source_tabular_result.get('evidence_envelopes') or [] + ) + agent_citations_list.extend( + mixed_source_tabular_result.get('agent_citations') or [] + ) + generated_tabular_outputs_list.extend( + mixed_source_tabular_result.get('generated_outputs') or [] + ) + generated_analysis_artifacts_list.extend( + mixed_source_tabular_result.get('generated_outputs') or [] + ) + mixed_source_handoff = build_mixed_source_evidence_handoff( + mixed_source_manifest, + mixed_source_evidence_envelopes, + effective_mixed_source_selection_mode, + ) + system_messages_for_augmentation.append(mixed_source_handoff) + mixed_source_coverage = mixed_source_handoff.get( + 'mixed_source_coverage', + {}, + ) + mixed_source_has_authorized_evidence_sources = bool( + mixed_source_narrative_document_ids + or mixed_source_tabular_sources + ) + user_metadata['mixed_source_coverage'] = mixed_source_coverage + user_message_doc['metadata'] = user_metadata + cosmos_messages_container.upsert_item(user_message_doc) + log_event( + '[MixedSourceChatSearch] Prepared streaming bounded mixed-source synthesis evidence.', + extra={ + 'selection_mode': effective_mixed_source_selection_mode, + 'narrative_result_count': len(search_results or []), + 'tabular_candidate_count': len(mixed_source_tabular_sources), + 'tabular_completed_count': sum( + 1 + for envelope in mixed_source_evidence_envelopes + if envelope.get('source_kind') == 'tabular' + and envelope.get('status') == 'completed' + ), + 'mixed_synthesis_count': 1, + 'partial_coverage': bool( + mixed_source_coverage.get('partial_coverage') + ), + }, + level=logging.INFO, + ) + + if ( + not is_mixed_source_chat_search_enabled(settings) + and (hybrid_search_enabled or history_grounded_search_used) + and workspace_tabular_files + and is_tabular_processing_enabled(settings) + ): tabular_source_hint = determine_tabular_source_hint( effective_document_scope, active_group_id=effective_active_group_id, @@ -18391,7 +19446,9 @@ def record_and_publish_streaming_thought(thought_payload): # Update message chat type message_chat_type = None - if (hybrid_search_enabled or history_grounded_search_used) and search_results and len(search_results) > 0: + if mixed_source_document_context_active and ( + search_results or mixed_source_has_authorized_evidence_sources + ): if effective_document_scope == 'group': message_chat_type = 'group' elif effective_document_scope == 'public': @@ -18403,8 +19460,10 @@ def record_and_publish_streaming_thought(thought_payload): source_review_used = _source_review_metadata_used(source_review_result) user_metadata['capability_usage'] = _build_capability_usage_metadata( - workspace_search_enabled=hybrid_search_enabled or history_grounded_search_used, - workspace_search_used=bool(search_results), + workspace_search_enabled=mixed_source_document_context_active, + workspace_search_used=bool( + search_results or mixed_source_has_authorized_evidence_sources + ), workspace_search_result_count=len(search_results or []), document_action_type=DOCUMENT_ACTION_TYPE_NONE, document_scope=effective_document_scope, @@ -18446,7 +19505,10 @@ def record_and_publish_streaming_thought(thought_payload): gpt_model=gpt_model, user_message_id=user_message_id, fallback_user_message=user_message, - include_assistant_citation_context=not explicit_external_retrieval_requested, + include_assistant_citation_context=( + not explicit_external_retrieval_requested + and not mixed_source_explicit_selection + ), ) summary_of_older = history_segments['summary_of_older'] chat_tabular_files = history_segments['chat_tabular_files'] @@ -18473,7 +19535,14 @@ def record_and_publish_streaming_thought(thought_payload): final_api_source_refs.extend(history_debug_info.get('history_message_source_refs', [])) # --- Mini SK analysis for tabular files uploaded directly to chat --- - if chat_tabular_files and is_tabular_processing_enabled(settings): + if ( + chat_tabular_files + and is_tabular_processing_enabled(settings) + and not ( + is_mixed_source_chat_search_enabled(settings) + and (mixed_source_explicit_selection or mixed_source_tabular_sources) + ) + ): chat_tabular_filenames_str = ", ".join(chat_tabular_files) chat_tabular_execution_mode = get_tabular_execution_mode(user_message) log_event( @@ -18645,6 +19714,10 @@ def record_and_publish_streaming_thought(thought_payload): original_hybrid_search_enabled, prior_grounded_document_refs, explicit_external_retrieval_requested, + current_document_context_requested=( + is_mixed_source_chat_search_enabled(settings) + and document_context_requested + ), ): history_grounding_message = build_history_grounding_system_message() insert_idx = 0 @@ -18987,6 +20060,8 @@ def finalize_cancelled_agent_stream_response(): 'document_scope': effective_document_scope, 'group_id': effective_active_group_id if chat_type == 'group' else None, 'hybrid_search_enabled': hybrid_search_enabled, + 'selection_mode': selection_mode, + 'document_context_requested': mixed_source_document_context_active, 'selected_document_id': effective_selected_document_id, 'selected_document_ids': effective_selected_document_ids, 'active_group_ids': effective_active_group_ids, @@ -19519,7 +20594,7 @@ def finalize_cancelled_agent_stream_response(): document_scope=effective_document_scope, selected_document_id=effective_selected_document_id, model_deployment=final_model_used if use_agent_streaming else gpt_model, - hybrid_search_enabled=hybrid_search_enabled or history_grounded_search_used, + hybrid_search_enabled=mixed_source_document_context_active, image_gen_enabled=False, selected_documents=combined_documents if combined_documents else None, selected_agent=agent_name_used if use_agent_streaming else None, @@ -20516,11 +21591,13 @@ def should_apply_history_grounding_message( original_hybrid_search_enabled, prior_grounded_document_refs, explicit_external_retrieval_requested=False, + current_document_context_requested=False, ): """Apply bounded grounding only when prior grounded docs exist for this conversation.""" return ( not bool(original_hybrid_search_enabled) and not bool(explicit_external_retrieval_requested) + and not bool(current_document_context_requested) and bool(prior_grounded_document_refs) ) diff --git a/application/single_app/route_backend_collaboration.py b/application/single_app/route_backend_collaboration.py index 1d00cec85..0733372e8 100644 --- a/application/single_app/route_backend_collaboration.py +++ b/application/single_app/route_backend_collaboration.py @@ -307,6 +307,8 @@ def _build_collaboration_stream_request_payload(data, source_conversation_id, me 'message': message_content, 'conversation_id': source_conversation_id, 'hybrid_search': bool(data.get('hybrid_search')), + 'selection_mode': data.get('selection_mode'), + 'document_context_requested': data.get('document_context_requested'), 'web_search_enabled': bool(data.get('web_search_enabled')), 'selected_document_id': data.get('selected_document_id'), 'selected_document_ids': data.get('selected_document_ids') or [], diff --git a/application/single_app/route_backend_conversations.py b/application/single_app/route_backend_conversations.py index d53fbb432..413de7bd1 100644 --- a/application/single_app/route_backend_conversations.py +++ b/application/single_app/route_backend_conversations.py @@ -171,6 +171,67 @@ def _normalize_workspace_document_delete_ids(raw_document_ids): return normalized_document_ids +def _build_replayed_document_context(original_metadata): + """Rebuild document-context intent from stored metadata for retry and edit.""" + metadata = original_metadata if isinstance(original_metadata, dict) else {} + workspace_search = metadata.get('workspace_search') + if not isinstance(workspace_search, dict): + workspace_search = metadata.get('document_search') + workspace_search = workspace_search if isinstance(workspace_search, dict) else {} + + selected_document_ids = ( + workspace_search.get('requested_document_ids') + or workspace_search.get('selected_document_ids') + or [] + ) + if not isinstance(selected_document_ids, list): + selected_document_ids = [selected_document_ids] + selected_document_ids = [ + str(document_id or '').strip() + for document_id in selected_document_ids + if str(document_id or '').strip() + ] + selected_document_id = str( + workspace_search.get('selected_document_id') + or workspace_search.get('document_id') + or '' + ).strip() + if selected_document_id and selected_document_id not in selected_document_ids: + selected_document_ids.insert(0, selected_document_id) + + selection_mode = str(workspace_search.get('selection_mode') or '').strip().lower() + if selection_mode not in {'selected', 'all', 'history', 'relevance'}: + selection_mode = 'selected' if selected_document_ids else 'relevance' + document_context_requested = workspace_search.get('document_context_requested') + if not isinstance(document_context_requested, bool): + document_context_requested = bool( + workspace_search.get('search_enabled') + or workspace_search.get('enabled') + or selected_document_ids + ) + + return { + 'hybrid_search': bool( + workspace_search.get('hybrid_search_preference') + if 'hybrid_search_preference' in workspace_search + else workspace_search.get('enabled') + or workspace_search.get('search_enabled') + ), + 'selection_mode': selection_mode, + 'document_context_requested': document_context_requested, + 'selected_document_id': selected_document_ids[0] if selected_document_ids else None, + 'selected_document_ids': selected_document_ids, + 'doc_scope': workspace_search.get('document_scope') or workspace_search.get('scope'), + 'top_n': workspace_search.get('top_n'), + 'classifications': ( + workspace_search.get('classification') + or workspace_search.get('classifications') + ), + 'active_group_ids': workspace_search.get('active_group_ids') or [], + 'active_public_workspace_ids': workspace_search.get('active_public_workspace_ids') or [], + } + + def _get_requested_workspace_document_delete_ids_for_conversation(payload, conversation_id): if not isinstance(payload, dict): return [] @@ -2641,16 +2702,13 @@ def retry_message(message_id): "message_retry_created", ) # Build chat request parameters from original message metadata + replayed_document_context = _build_replayed_document_context(original_metadata) chat_request = { 'message': user_content, 'conversation_id': conversation_id, 'model_deployment': selected_model or original_metadata.get('model_selection', {}).get('selected_model'), 'reasoning_effort': reasoning_effort or original_metadata.get('reasoning_effort'), - 'hybrid_search': original_metadata.get('document_search', {}).get('enabled', False), - 'selected_document_id': original_metadata.get('document_search', {}).get('document_id'), - 'doc_scope': original_metadata.get('document_search', {}).get('scope'), - 'top_n': original_metadata.get('document_search', {}).get('top_n'), - 'classifications': original_metadata.get('document_search', {}).get('classifications'), + **replayed_document_context, 'image_generation': original_metadata.get('image_generation', {}).get('enabled', False), 'active_group_id': original_metadata.get('chat_context', {}).get('group_id'), 'active_public_workspace_id': original_metadata.get('chat_context', {}).get('public_workspace_id'), @@ -2864,16 +2922,13 @@ def edit_message(message_id): ) # Build chat request parameters from original message metadata # Keep all original settings (model, reasoning, doc search, etc.) + replayed_document_context = _build_replayed_document_context(original_metadata) chat_request = { 'message': edited_content, # Use edited content 'conversation_id': conversation_id, 'model_deployment': original_metadata.get('model_selection', {}).get('selected_model'), 'reasoning_effort': original_metadata.get('reasoning_effort'), - 'hybrid_search': original_metadata.get('document_search', {}).get('enabled', False), - 'selected_document_id': original_metadata.get('document_search', {}).get('document_id'), - 'doc_scope': original_metadata.get('document_search', {}).get('scope'), - 'top_n': original_metadata.get('document_search', {}).get('top_n'), - 'classifications': original_metadata.get('document_search', {}).get('classifications'), + **replayed_document_context, 'image_generation': original_metadata.get('image_generation', {}).get('enabled', False), 'active_group_id': original_metadata.get('chat_context', {}).get('group_id'), 'active_public_workspace_id': original_metadata.get('chat_context', {}).get('public_workspace_id'), diff --git a/application/single_app/semantic_kernel_plugins/tabular_processing_plugin.py b/application/single_app/semantic_kernel_plugins/tabular_processing_plugin.py index c038cd0ce..a041c12f1 100644 --- a/application/single_app/semantic_kernel_plugins/tabular_processing_plugin.py +++ b/application/single_app/semantic_kernel_plugins/tabular_processing_plugin.py @@ -234,6 +234,14 @@ def _get_authorized_chat_context(self) -> dict: for workspace_id in (authorized_context.get('active_public_workspace_ids') or []) if str(workspace_id or '').strip() ] + authorized_blob_locations = [ + [str(location[0]), str(location[1])] + for location in authorized_context.get('authorized_blob_locations') or [] + if isinstance(location, (list, tuple)) + and len(location) == 2 + and str(location[0] or '').strip() + and str(location[1] or '').strip() + ] return { 'user_id': authorized_user_id, @@ -244,6 +252,7 @@ def _get_authorized_chat_context(self) -> dict: 'active_public_workspace_id': ( str(authorized_context.get('active_public_workspace_id') or '').strip() or None ), + 'authorized_blob_locations': authorized_blob_locations, } def _resolve_authorized_scope_arguments( @@ -346,6 +355,14 @@ def _is_authorized_public_workspace_scope( def _is_authorized_blob_location(self, container_name: str, blob_path: str, authorized_context: dict) -> bool: """Ensure remembered blob locations still fall within the caller's authorized request scope.""" + exact_authorized_locations = { + (str(location[0]), str(location[1])) + for location in authorized_context.get('authorized_blob_locations') or [] + if isinstance(location, (list, tuple)) and len(location) == 2 + } + if (str(container_name or ''), str(blob_path or '')) in exact_authorized_locations: + return True + source = self._infer_source_from_container(container_name) blob_parts = [part for part in str(blob_path or '').split('/') if part] if not source or not blob_parts: diff --git a/application/single_app/static/js/chat/chat-messages.js b/application/single_app/static/js/chat/chat-messages.js index 7c5ba84f9..3dd73e4a4 100644 --- a/application/single_app/static/js/chat/chat-messages.js +++ b/application/single_app/static/js/chat/chat-messages.js @@ -6102,6 +6102,8 @@ export function buildChatRequestPayload(finalMessageToSend, conversationId = cur .filter(value => value); selectedDocumentId = selectedDocumentIds.length > 0 ? selectedDocumentIds[0] : null; } + const selectionMode = selectedDocumentIds.length > 0 ? 'selected' : 'relevance'; + const documentContextRequested = hybridSearchEnabled || selectedDocumentIds.length > 0; let imageGenEnabled = false; const igbtn = document.getElementById('image-generate-btn'); @@ -6216,6 +6218,8 @@ export function buildChatRequestPayload(finalMessageToSend, conversationId = cur message: finalMessageToSend, conversation_id: conversationId, hybrid_search: hybridSearchEnabled, + selection_mode: selectionMode, + document_context_requested: documentContextRequested, user_workspace_context_enabled: userWorkspaceContextEnabled, web_search_enabled: webSearchEnabled, url_access_enabled: urlAccessEnabled, @@ -6277,6 +6281,13 @@ export function buildCollaborativeInvocationTarget(messageData = {}, explicitInv messageData.agent_info && (messageData.agent_info.id || messageData.agent_info.name || messageData.agent_info.display_name) ); + const workspaceContextInvocationRequested = Boolean( + messageData.hybrid_search + || ( + messageData.document_context_requested + && window.appSettings?.enable_mixed_source_chat_search + ) + ); const sourceMode = messageData.image_generation ? 'image_generation' : hasAgentTarget @@ -6287,7 +6298,7 @@ export function buildCollaborativeInvocationTarget(messageData = {}, explicitInv ? 'url_access' : messageData.web_search_enabled ? 'web_search' - : messageData.hybrid_search + : workspaceContextInvocationRequested ? 'workspace' : messageData.prompt_info ? 'prompt' diff --git a/application/single_app/templates/chats.html b/application/single_app/templates/chats.html index 460a49b16..38dd6d3eb 100644 --- a/application/single_app/templates/chats.html +++ b/application/single_app/templates/chats.html @@ -1487,6 +1487,7 @@