Support generated JSON and XML exports

application/single_app/config.py

@@ -95,7 +95,7 @@
 EXECUTOR_TYPE = 'thread'
 EXECUTOR_MAX_WORKERS = 30
 SESSION_TYPE = 'filesystem'
-VERSION = "0.250.112"
+VERSION = "0.250.114"
 IS_DEVELOPMENT = is_development_env_enabled()
 
 SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax')

application/single_app/functions_document_analysis.py

@@ -363,6 +363,119 @@
     return f"{document_name} - window {window_range.get('window_number')} ({range_label})"
 
 
+def _prompt_requests_json_output(analysis_prompt):
+    prompt_text = str(analysis_prompt or '').strip().lower()
+    if not prompt_text:
+        return False
+
+    json_markers = (
+        'json artifact',
+        'json export',
+        'json output',
+        'json array',
+        'json object',
+        'json file',
+        'json format',
+        'valid json',
+        'convert into json',
+        'convert to json',
+        'return json',
+        'return only json',
+        'respond with json',
+        'format as json',
+        'output as json',
+        'save as json',
+        'export as json',
+        'download as json',
+        'create json',
+        'create a json',
+        'make json',
+        'make a json',
+        'generate json',
+        'generate a json',
+    )
+    if any(marker in prompt_text for marker in json_markers):
+        return True
+
+    return bool(re.search(
+        r'\b(convert|create|make|build|generate|produce|return|respond|format|output|save|export|download)\b[\w\s.,:;\-/]{0,80}\bjson\b',
+        prompt_text,
+    ))
+
+
+def _prompt_requests_xml_output(analysis_prompt):
+    prompt_text = str(analysis_prompt or '').strip().lower()
+    if not prompt_text:
+        return False
+
+    xml_markers = (
+        'xml artifact',
+        'xml export',
+        'xml output',
+        'xml document',
+        'xml file',
+        'xml template',
+        'valid xml',
+        'well-formed xml',
+        'convert into xml',
+        'convert to xml',
+        'populate xml',
+        'populate the xml',
+        'return xml',
+        'return only xml',
+        'respond with xml',
+        'format as xml',
+        'output as xml',
+        'save as xml',
+        'export as xml',
+        'download as xml',
+        'create xml',
+        'create an xml',
+        'make xml',
+        'make an xml',
+        'generate xml',
+        'generate an xml',
+    )
+    if any(marker in prompt_text for marker in xml_markers):
+        return True
+
+    return bool(re.search(
+        r'\b(convert|populate|create|make|build|generate|produce|return|respond|format|output|save|export|download)\b[\w\s.,:;\-/]{0,80}\bxml\b',
+        prompt_text,
+    ))
+
+
+def _build_requested_output_guidance(analysis_prompt, stage):
+    if _prompt_requests_xml_output(analysis_prompt):
+        if stage == 'slice':
+            return (
+                'The overall task requests XML output. For this slice, preserve exact XML element names, '
+                'attribute names, nesting, template placeholders, and source values needed to produce the final XML. '
+                'Do not condense repeated XML structures when they are visible in this slice. If this slice contains '
+                'everything needed to satisfy the task, return only the complete well-formed XML document.\n\n'
+            )
+        return (
+            'The original task requests an XML file. Return only one complete well-formed XML document for the final '
+            'answer, without Markdown fences, prose, citations, or explanatory text outside the XML. Preserve the '
+            'requested template structure whenever a template is supplied.\n\n'
+        )
+
+    if _prompt_requests_json_output(analysis_prompt):
+        if stage == 'slice':
+            return (
+                'The overall task requests JSON output. For this slice, preserve exact field names, hierarchy, arrays, '
+                'template placeholders, and source values needed to produce the final JSON. Do not condense repeated '
+                'structures when they are visible in this slice. If this slice contains everything needed to satisfy '
+                'the task, return only valid JSON.\n\n'
+            )
+        return (
+            'The original task requests a JSON file. Return only valid JSON for the final answer, without Markdown '
+            'fences, prose, citations, or explanatory text outside the JSON.\n\n'
+        )
+
+    return ''
+
+
 def _build_window_analysis_prompt(analysis_prompt, document_payload, window_payload, window_range):
     document_file_name = _resolve_document_file_name(document_payload)
     document_title = _resolve_document_title(document_payload)
@@ -387,6 +500,7 @@
         f'Page count in slice: {window_range.get("page_count", 0)}\n\n'
         'Task instructions:\n'
         f'{analysis_prompt}\n\n'
+        f'{_build_requested_output_guidance(analysis_prompt, "slice")}'
         'Write a focused analysis of this slice. Preserve concrete facts, decisions, comments, action items, '
         'and open questions. Call out anything that still needs follow-up.\n\n'
         f'<DocumentSlice>\n{_render_window_source_text(window_payload)}\n</DocumentSlice>'
@@ -508,11 +622,15 @@
 
 def _build_analysis_intent(analysis_prompt):
     per_source_output_requested = _prompt_requests_per_source_output(analysis_prompt)
+    json_output_requested = _prompt_requests_json_output(analysis_prompt)
+    xml_output_requested = _prompt_requests_xml_output(analysis_prompt)
     json_array_output_requested = _prompt_requests_json_array_output(analysis_prompt)
     json_code_block_requested = _prompt_requests_json_code_block(analysis_prompt)
     table_output_requested = _prompt_requests_table_output(analysis_prompt)
     exhaustive_output_requested = (
         per_source_output_requested
+        or json_output_requested
+        or xml_output_requested
         or json_array_output_requested
         or table_output_requested
         or _prompt_requests_exhaustive_output(analysis_prompt)
@@ -522,11 +640,13 @@
         'exhaustive': exhaustive_output_requested,
         'preserve_raw_outputs': True,
         'per_source_output_requested': per_source_output_requested,
+        'json_output_requested': json_output_requested,
+        'xml_output_requested': xml_output_requested,
         'json_array_output_requested': json_array_output_requested,
         'json_code_block_requested': json_code_block_requested,
         'table_output_requested': table_output_requested,
-        'csv_artifact_recommended': table_output_requested or exhaustive_output_requested,
-        'markdown_analysis_artifact_recommended': exhaustive_output_requested,
+        'csv_artifact_recommended': table_output_requested or (exhaustive_output_requested and not json_output_requested and not xml_output_requested),
+        'markdown_analysis_artifact_recommended': exhaustive_output_requested and not json_output_requested and not xml_output_requested,
     }
 
 
@@ -627,6 +747,7 @@
         f'Task instructions:\n{analysis_prompt}\n\n'
         f'{failed_note}'
         f'{preservation_note}'
+        f'{_build_requested_output_guidance(analysis_prompt, "reduction")}'
         f'{combine_instruction}\n\n'
         f'<WindowAnalyses>\n{combined_text}\n</WindowAnalyses>'
     )
@@ -659,6 +780,7 @@
         f'Source document: {document_name}\n'
         f'Task instructions:\n{analysis_prompt}\n\n'
         f'{failed_note}'
+        f'{_build_requested_output_guidance(analysis_prompt, "reduction")}'
         'Combine the slice analyses below into one document-level answer.\n\n'
         f'<DocumentWindowAnalyses>\n{combined_text}\n</DocumentWindowAnalyses>'
     )

application/single_app/functions_documents.py

@@ -5159,7 +5159,7 @@ def process_txt(document_id, user_id, temp_file_path, original_filename, enable_
 
     return total_chunks_saved, total_embedding_tokens, embedding_model_name
 
-def process_xml(document_id, user_id, temp_file_path, original_filename, enable_enhanced_citations, update_callback, group_id=None, public_workspace_id=None):
+def _process_xml_with_token_usage(document_id, user_id, temp_file_path, original_filename, enable_enhanced_citations, update_callback, group_id=None, public_workspace_id=None):
     """Processes XML files using RecursiveCharacterTextSplitter for structured content."""
     is_group = group_id is not None
     is_public_workspace = public_workspace_id is not None
@@ -5215,7 +5215,16 @@ def process_xml(document_id, user_id, temp_file_path, original_filename, enable_
         for idx, chunk_content in enumerate(final_chunks, start=1):
             # Skip empty chunks
             if not chunk_content or not chunk_content.strip():
-                print(f"Skipping empty XML chunk {idx}/{initial_chunk_count}")
+                log_event(
+                    '[Documents] Skipping empty XML chunk',
+                    {
+                        'document_id': document_id,
+                        'file_name': original_filename,
+                        'chunk_index': idx,
+                        'chunk_count': initial_chunk_count,
+                    },
+                    debug_only=True,
+                )
                 continue
 
             update_callback(
@@ -5247,10 +5256,29 @@ def process_xml(document_id, user_id, temp_file_path, original_filename, enable_
         # Final update with actual chunks saved
         if total_chunks_saved != initial_chunk_count:
             update_callback(number_of_pages=total_chunks_saved)
-            print(f"Adjusted final chunk count from {initial_chunk_count} to {total_chunks_saved} after skipping empty chunks.")
+            log_event(
+                '[Documents] Adjusted XML chunk count after skipping empty chunks',
+                {
+                    'document_id': document_id,
+                    'file_name': original_filename,
+                    'initial_chunk_count': initial_chunk_count,
+                    'total_chunks_saved': total_chunks_saved,
+                },
+                debug_only=True,
+            )
 
     except Exception as e:
-        print(f"Error during XML processing for {original_filename}: {type(e).__name__}: {e}")
+        log_event(
+            '[Documents] XML processing failed',
+            {
+                'document_id': document_id,
+                'file_name': original_filename,
+                'error_type': type(e).__name__,
+                'error': str(e),
+            },
+            level=logging.ERROR,
+            exceptionTraceback=True,
+        )
         raise Exception(f"Failed processing XML file {original_filename}: {e}")
 
     return total_chunks_saved, total_embedding_tokens, embedding_model_name
@@ -5539,92 +5567,17 @@ def process_doc(document_id, user_id, temp_file_path, original_filename, enable_
     return total_chunks_saved, total_embedding_tokens, embedding_model_name
 
 def process_xml(document_id, user_id, temp_file_path, original_filename, enable_enhanced_citations, update_callback, group_id=None, public_workspace_id=None):
-    """Processes XML files using RecursiveCharacterTextSplitter for structured content."""
-    is_group = group_id is not None
-    is_public_workspace = public_workspace_id is not None
-
-    update_callback(status="Processing XML file...")
-    total_chunks_saved = 0
-    # Character-based chunking for XML structure preservation, capped by embedding context
-    chunk_config = get_chunk_size_config(get_settings())
-    max_chunk_size_chars = chunk_config.get('xml', {}).get('value', 4000)
-
-    if enable_enhanced_citations:
-        args = {
-            "temp_file_path": temp_file_path,
-            "user_id": user_id,
-            "document_id": document_id,
-            "blob_filename": original_filename,
-            "update_callback": update_callback
-        }
-
-        if is_group:
-            args["group_id"] = group_id
-        elif is_public_workspace:
-            args["public_workspace_id"] = public_workspace_id
-
-        upload_to_blob(**args)
-
-    try:
-        # Read XML content
-        try:
-            with open(temp_file_path, 'r', encoding='utf-8') as f:
-                xml_content = f.read()
-        except Exception as e:
-            raise Exception(f"Error reading XML file {original_filename}: {e}")
-
-        # Use RecursiveCharacterTextSplitter with XML-aware separators
-        # This preserves XML structure better than simple word splitting
-        xml_splitter = RecursiveCharacterTextSplitter(
-            chunk_size=max_chunk_size_chars,
-            chunk_overlap=0,
-            length_function=len,
-            separators=["\n\n", "\n", ">", " ", ""],  # XML-friendly separators
-            is_separator_regex=False
-        )
-
-        # Split the XML content
-        final_chunks = xml_splitter.split_text(xml_content)
-
-        initial_chunk_count = len(final_chunks)
-        update_callback(number_of_pages=initial_chunk_count)
-
-        for idx, chunk_content in enumerate(final_chunks, start=1):
-            # Skip empty chunks
-            if not chunk_content or not chunk_content.strip():
-                print(f"Skipping empty XML chunk {idx}/{initial_chunk_count}")
-                continue
-
-            update_callback(
-                current_file_chunk=idx,
-                status=f"Saving chunk {idx}/{initial_chunk_count}..."
-            )
-            args = {
-                "page_text_content": chunk_content,
-                "page_number": total_chunks_saved + 1,
-                "file_name": original_filename,
-                "user_id": user_id,
-                "document_id": document_id
-            }
-
-            if is_public_workspace:
-                args["public_workspace_id"] = public_workspace_id
-            elif is_group:
-                args["group_id"] = group_id
-
-            save_chunks(**args)
-            total_chunks_saved += 1
-
-        # Final update with actual chunks saved
-        if total_chunks_saved != initial_chunk_count:
-            update_callback(number_of_pages=total_chunks_saved)
-            print(f"Adjusted final chunk count from {initial_chunk_count} to {total_chunks_saved} after skipping empty chunks.")
-
-    except Exception as e:
-        print(f"Error during XML processing for {original_filename}: {type(e).__name__}: {e}")
-        raise Exception(f"Failed processing XML file {original_filename}: {e}")
-
-    return total_chunks_saved
+    """Processes XML files using the consolidated token-aware XML pipeline."""
+    return _process_xml_with_token_usage(
+        document_id,
+        user_id,
+        temp_file_path,
+        original_filename,
+        enable_enhanced_citations,
+        update_callback,
+        group_id=group_id,
+        public_workspace_id=public_workspace_id,
+    )
 
 def process_yaml(document_id, user_id, temp_file_path, original_filename, enable_enhanced_citations, update_callback, group_id=None, public_workspace_id=None):
     """Processes YAML files using RecursiveCharacterTextSplitter for structured content."""