From 323b5519cb376efb3b79a78039c41d72e236fc34 Mon Sep 17 00:00:00 2001 From: shaohuzhang1 Date: Wed, 16 Sep 2026 11:37:57 +0800 Subject: [PATCH] feat: Completely clear out the old workflow engine. --- apps/application/flow/__init__.py | 8 - apps/application/flow/backend/__init__.py | 0 .../application/flow/backend/sandbox_shell.py | 311 ------ apps/application/flow/common.py | 284 ------ apps/application/flow/compare/__init__.py | 83 -- apps/application/flow/compare/compare.py | 15 - .../flow/compare/contain_compare.py | 25 - apps/application/flow/compare/end_with.py | 16 - .../application/flow/compare/equal_compare.py | 15 - apps/application/flow/compare/ge_compare.py | 25 - apps/application/flow/compare/gt_compare.py | 25 - .../flow/compare/is_not_null_compare.py | 18 - apps/application/flow/compare/is_not_true.py | 18 - .../flow/compare/is_null_compare.py | 18 - apps/application/flow/compare/is_true.py | 18 - apps/application/flow/compare/le_compare.py | 25 - .../flow/compare/len_equal_compare.py | 18 - .../flow/compare/len_ge_compare.py | 18 - .../flow/compare/len_gt_compare.py | 18 - .../flow/compare/len_le_compare.py | 18 - .../flow/compare/len_lt_compare.py | 18 - apps/application/flow/compare/lt_compare.py | 25 - .../flow/compare/not_contain_compare.py | 25 - .../flow/compare/not_equal_compare.py | 15 - .../application/flow/compare/regex_compare.py | 35 - apps/application/flow/compare/start_with.py | 16 - .../flow/compare/wildcard_compare.py | 38 - apps/application/flow/default_workflow.json | 451 --------- .../application/flow/default_workflow_en.json | 451 --------- .../application/flow/default_workflow_zh.json | 451 --------- .../flow/default_workflow_zh_Hant.json | 451 --------- apps/application/flow/i_step_node.py | 470 --------- .../flow/knowledge_loop_workflow_manage.py | 21 - .../flow/knowledge_workflow_manage.py | 130 --- apps/application/flow/loop_workflow_manage.py | 199 ---- apps/application/flow/step_node/__init__.py | 63 -- .../step_node/ai_chat_step_node/__init__.py | 9 - .../ai_chat_step_node/i_chat_node.py | 92 -- .../ai_chat_step_node/impl/__init__.py | 9 - .../ai_chat_step_node/impl/base_chat_node.py | 624 ------------ .../step_node/application_node/__init__.py | 2 - .../application_node/i_application_node.py | 106 -- .../application_node/impl/__init__.py | 2 - .../impl/base_application_node.py | 299 ------ .../flow/step_node/condition_node/__init__.py | 9 - .../condition_node/i_condition_node.py | 42 - .../step_node/condition_node/impl/__init__.py | 9 - .../impl/base_condition_node.py | 47 - .../data_source_local_node/__init__.py | 8 - .../i_data_source_local_node.py | 42 - .../data_source_local_node/impl/__init__.py | 8 - .../impl/base_data_source_local_node.py | 52 - .../data_source_web_node/__init__.py | 8 - .../i_data_source_web_node.py | 28 - .../data_source_web_node/impl/__init__.py | 8 - .../impl/base_data_source_web_node.py | 98 -- .../step_node/direct_reply_node/__init__.py | 9 - .../direct_reply_node/i_reply_node.py | 58 -- .../direct_reply_node/impl/__init__.py | 9 - .../direct_reply_node/impl/base_reply_node.py | 47 - .../document_extract_node/__init__.py | 1 - .../i_document_extract_node.py | 30 - .../document_extract_node/impl/__init__.py | 1 - .../impl/base_document_extract_node.py | 95 -- .../step_node/document_split_node/__init__.py | 1 - .../i_document_split_node.py | 97 -- .../document_split_node/impl/__init__.py | 1 - .../impl/base_document_split_node.py | 192 ---- .../flow/step_node/form_node/__init__.py | 9 - .../flow/step_node/form_node/i_form_node.py | 37 - .../flow/step_node/form_node/impl/__init__.py | 9 - .../form_node/impl/base_form_node.py | 238 ----- .../image_generate_step_node/__init__.py | 3 - .../i_image_generate_node.py | 56 -- .../image_generate_step_node/impl/__init__.py | 3 - .../impl/base_image_generate_node.py | 199 ---- .../image_to_video_step_node/__init__.py | 3 - .../i_image_to_video_node.py | 78 -- .../image_to_video_step_node/impl/__init__.py | 3 - .../impl/base_image_to_video_node.py | 213 ---- .../image_understand_step_node/__init__.py | 3 - .../i_image_understand_node.py | 63 -- .../impl/__init__.py | 3 - .../impl/base_image_understand_node.py | 340 ------- .../flow/step_node/intent_node/__init__.py | 6 - .../step_node/intent_node/i_intent_node.py | 59 -- .../step_node/intent_node/impl/__init__.py | 3 - .../intent_node/impl/base_intent_node.py | 266 ----- .../intent_node/impl/prompt_template.py | 29 - .../knowledge_write_node/__init__.py | 8 - .../i_knowledge_write_node.py | 43 - .../knowledge_write_node/impl/__init__.py | 8 - .../impl/base_knowledge_write_node.py | 380 -------- .../step_node/loop_break_node/__init__.py | 9 - .../loop_break_node/i_loop_break_node.py | 41 - .../loop_break_node/impl/__init__.py | 9 - .../impl/base_loop_break_node.py | 47 - .../step_node/loop_continue_node/__init__.py | 9 - .../i_loop_continue_node.py | 40 - .../loop_continue_node/impl/__init__.py | 9 - .../impl/base_loop_continue_node.py | 35 - .../flow/step_node/loop_node/__init__.py | 9 - .../flow/step_node/loop_node/i_loop_node.py | 58 -- .../flow/step_node/loop_node/impl/__init__.py | 9 - .../loop_node/impl/base_loop_node.py | 332 ------- .../step_node/loop_start_node/__init__.py | 9 - .../loop_start_node/i_loop_start_node.py | 21 - .../loop_start_node/impl/__init__.py | 9 - .../loop_start_node/impl/base_start_node.py | 59 -- .../flow/step_node/mcp_node/__init__.py | 3 - .../flow/step_node/mcp_node/i_mcp_node.py | 33 - .../flow/step_node/mcp_node/impl/__init__.py | 3 - .../step_node/mcp_node/impl/base_mcp_node.py | 72 -- .../parameter_extraction_node/__init__.py | 9 - .../i_parameter_extraction_node.py | 58 -- .../impl/__init__.py | 9 - .../impl/base_parameter_extraction_node.py | 124 --- .../flow/step_node/question_node/__init__.py | 9 - .../question_node/i_question_node.py | 55 -- .../step_node/question_node/impl/__init__.py | 9 - .../question_node/impl/base_question_node.py | 173 ---- .../flow/step_node/reranker_node/__init__.py | 9 - .../reranker_node/i_reranker_node.py | 84 -- .../step_node/reranker_node/impl/__init__.py | 9 - .../reranker_node/impl/base_reranker_node.py | 129 --- .../search_document_node/__init__.py | 1 - .../i_search_document_node.py | 58 -- .../search_document_node/impl/__init__.py | 1 - .../impl/base_search_document_node.py | 212 ---- .../search_knowledge_node/__init__.py | 9 - .../i_search_knowledge_node.py | 96 -- .../search_knowledge_node/impl/__init__.py | 9 - .../impl/base_search_knowledge_node.py | 226 ----- .../speech_to_text_step_node/__init__.py | 3 - .../i_speech_to_text_node.py | 47 - .../speech_to_text_step_node/impl/__init__.py | 3 - .../impl/base_speech_to_text_node.py | 89 -- .../flow/step_node/start_node/__init__.py | 9 - .../flow/step_node/start_node/i_start_node.py | 21 - .../step_node/start_node/impl/__init__.py | 9 - .../start_node/impl/base_start_node.py | 121 --- .../text_to_speech_step_node/__init__.py | 3 - .../i_text_to_speech_node.py | 42 - .../text_to_speech_step_node/impl/__init__.py | 3 - .../impl/base_text_to_speech_node.py | 178 ---- .../text_to_video_step_node/__init__.py | 3 - .../i_text_to_video_node.py | 57 -- .../text_to_video_step_node/impl/__init__.py | 3 - .../impl/base_text_to_video_node.py | 188 ---- .../flow/step_node/tool_lib_node/__init__.py | 9 - .../tool_lib_node/i_tool_lib_node.py | 54 -- .../step_node/tool_lib_node/impl/__init__.py | 9 - .../tool_lib_node/impl/base_tool_lib_node.py | 311 ------ .../flow/step_node/tool_node/__init__.py | 9 - .../flow/step_node/tool_node/i_tool_node.py | 66 -- .../flow/step_node/tool_node/impl/__init__.py | 9 - .../tool_node/impl/base_tool_node.py | 129 --- .../step_node/tool_start_node/__init__.py | 9 - .../tool_start_node/i_tool_start_node.py | 21 - .../tool_start_node/impl/__init__.py | 9 - .../impl/base_tool_start_node.py | 66 -- .../tool_workflow_lib_node/__init__.py | 9 - .../i_tool_workflow_lib_node.py | 57 -- .../tool_workflow_lib_node/impl/__init__.py | 9 - .../impl/base_tool_workflow_lib_node.py | 256 ----- .../variable_aggregation_node/__init__.py | 0 .../i_variable_aggregation_node.py | 42 - .../impl/__init__.py | 0 .../impl/base_variable_aggregation_node.py | 98 -- .../variable_assign_node/__init__.py | 3 - .../i_variable_assign_node.py | 29 - .../variable_assign_node/impl/__init__.py | 9 - .../impl/base_variable_assign_node.py | 125 --- .../variable_splitting_node/__init__.py | 9 - .../i_variable_splitting_node.py | 35 - .../variable_splitting_node/impl/__init__.py | 9 - .../impl/base_variable_splitting_node.py | 80 -- .../video_understand_step_node/__init__.py | 3 - .../i_video_understand_node.py | 63 -- .../impl/__init__.py | 3 - .../impl/base_video_understand_node.py | 335 ------- .../flow/tool_loop_workflow_manage.py | 21 - apps/application/flow/tool_workflow_manage.py | 88 -- apps/application/flow/tools.py | 911 ------------------ apps/application/flow/workflow_manage.py | 833 ---------------- apps/application/serializers/application.py | 4 +- .../{flow => workflow}/backend/sandbox_mcp.py | 0 apps/chat/serializers/chat.py | 2 +- apps/common/mcp/client.py | 2 +- apps/common/utils/common.py | 10 + apps/knowledge/tests.py | 21 +- apps/tools/serializers/tool.py | 4 +- apps/tools/serializers/tool_workflow.py | 2 +- .../impl/task/tool_task/workflow_tool_task.py | 132 +-- 194 files changed, 103 insertions(+), 14626 deletions(-) delete mode 100644 apps/application/flow/__init__.py delete mode 100644 apps/application/flow/backend/__init__.py delete mode 100644 apps/application/flow/backend/sandbox_shell.py delete mode 100644 apps/application/flow/common.py delete mode 100644 apps/application/flow/compare/__init__.py delete mode 100644 apps/application/flow/compare/compare.py delete mode 100644 apps/application/flow/compare/contain_compare.py delete mode 100644 apps/application/flow/compare/end_with.py delete mode 100644 apps/application/flow/compare/equal_compare.py delete mode 100644 apps/application/flow/compare/ge_compare.py delete mode 100644 apps/application/flow/compare/gt_compare.py delete mode 100644 apps/application/flow/compare/is_not_null_compare.py delete mode 100644 apps/application/flow/compare/is_not_true.py delete mode 100644 apps/application/flow/compare/is_null_compare.py delete mode 100644 apps/application/flow/compare/is_true.py delete mode 100644 apps/application/flow/compare/le_compare.py delete mode 100644 apps/application/flow/compare/len_equal_compare.py delete mode 100644 apps/application/flow/compare/len_ge_compare.py delete mode 100644 apps/application/flow/compare/len_gt_compare.py delete mode 100644 apps/application/flow/compare/len_le_compare.py delete mode 100644 apps/application/flow/compare/len_lt_compare.py delete mode 100644 apps/application/flow/compare/lt_compare.py delete mode 100644 apps/application/flow/compare/not_contain_compare.py delete mode 100644 apps/application/flow/compare/not_equal_compare.py delete mode 100644 apps/application/flow/compare/regex_compare.py delete mode 100644 apps/application/flow/compare/start_with.py delete mode 100644 apps/application/flow/compare/wildcard_compare.py delete mode 100644 apps/application/flow/default_workflow.json delete mode 100644 apps/application/flow/default_workflow_en.json delete mode 100644 apps/application/flow/default_workflow_zh.json delete mode 100644 apps/application/flow/default_workflow_zh_Hant.json delete mode 100644 apps/application/flow/i_step_node.py delete mode 100644 apps/application/flow/knowledge_loop_workflow_manage.py delete mode 100644 apps/application/flow/knowledge_workflow_manage.py delete mode 100644 apps/application/flow/loop_workflow_manage.py delete mode 100644 apps/application/flow/step_node/__init__.py delete mode 100644 apps/application/flow/step_node/ai_chat_step_node/__init__.py delete mode 100644 apps/application/flow/step_node/ai_chat_step_node/i_chat_node.py delete mode 100644 apps/application/flow/step_node/ai_chat_step_node/impl/__init__.py delete mode 100644 apps/application/flow/step_node/ai_chat_step_node/impl/base_chat_node.py delete mode 100644 apps/application/flow/step_node/application_node/__init__.py delete mode 100644 apps/application/flow/step_node/application_node/i_application_node.py delete mode 100644 apps/application/flow/step_node/application_node/impl/__init__.py delete mode 100644 apps/application/flow/step_node/application_node/impl/base_application_node.py delete mode 100644 apps/application/flow/step_node/condition_node/__init__.py delete mode 100644 apps/application/flow/step_node/condition_node/i_condition_node.py delete mode 100644 apps/application/flow/step_node/condition_node/impl/__init__.py delete mode 100644 apps/application/flow/step_node/condition_node/impl/base_condition_node.py delete mode 100644 apps/application/flow/step_node/data_source_local_node/__init__.py delete mode 100644 apps/application/flow/step_node/data_source_local_node/i_data_source_local_node.py delete mode 100644 apps/application/flow/step_node/data_source_local_node/impl/__init__.py delete mode 100644 apps/application/flow/step_node/data_source_local_node/impl/base_data_source_local_node.py delete mode 100644 apps/application/flow/step_node/data_source_web_node/__init__.py delete mode 100644 apps/application/flow/step_node/data_source_web_node/i_data_source_web_node.py delete mode 100644 apps/application/flow/step_node/data_source_web_node/impl/__init__.py delete mode 100644 apps/application/flow/step_node/data_source_web_node/impl/base_data_source_web_node.py delete mode 100644 apps/application/flow/step_node/direct_reply_node/__init__.py delete mode 100644 apps/application/flow/step_node/direct_reply_node/i_reply_node.py delete mode 100644 apps/application/flow/step_node/direct_reply_node/impl/__init__.py delete mode 100644 apps/application/flow/step_node/direct_reply_node/impl/base_reply_node.py delete mode 100644 apps/application/flow/step_node/document_extract_node/__init__.py delete mode 100644 apps/application/flow/step_node/document_extract_node/i_document_extract_node.py delete mode 100644 apps/application/flow/step_node/document_extract_node/impl/__init__.py delete mode 100644 apps/application/flow/step_node/document_extract_node/impl/base_document_extract_node.py delete mode 100644 apps/application/flow/step_node/document_split_node/__init__.py delete mode 100644 apps/application/flow/step_node/document_split_node/i_document_split_node.py delete mode 100644 apps/application/flow/step_node/document_split_node/impl/__init__.py delete mode 100644 apps/application/flow/step_node/document_split_node/impl/base_document_split_node.py delete mode 100644 apps/application/flow/step_node/form_node/__init__.py delete mode 100644 apps/application/flow/step_node/form_node/i_form_node.py delete mode 100644 apps/application/flow/step_node/form_node/impl/__init__.py delete mode 100644 apps/application/flow/step_node/form_node/impl/base_form_node.py delete mode 100644 apps/application/flow/step_node/image_generate_step_node/__init__.py delete mode 100644 apps/application/flow/step_node/image_generate_step_node/i_image_generate_node.py delete mode 100644 apps/application/flow/step_node/image_generate_step_node/impl/__init__.py delete mode 100644 apps/application/flow/step_node/image_generate_step_node/impl/base_image_generate_node.py delete mode 100644 apps/application/flow/step_node/image_to_video_step_node/__init__.py delete mode 100644 apps/application/flow/step_node/image_to_video_step_node/i_image_to_video_node.py delete mode 100644 apps/application/flow/step_node/image_to_video_step_node/impl/__init__.py delete mode 100644 apps/application/flow/step_node/image_to_video_step_node/impl/base_image_to_video_node.py delete mode 100644 apps/application/flow/step_node/image_understand_step_node/__init__.py delete mode 100644 apps/application/flow/step_node/image_understand_step_node/i_image_understand_node.py delete mode 100644 apps/application/flow/step_node/image_understand_step_node/impl/__init__.py delete mode 100644 apps/application/flow/step_node/image_understand_step_node/impl/base_image_understand_node.py delete mode 100644 apps/application/flow/step_node/intent_node/__init__.py delete mode 100644 apps/application/flow/step_node/intent_node/i_intent_node.py delete mode 100644 apps/application/flow/step_node/intent_node/impl/__init__.py delete mode 100644 apps/application/flow/step_node/intent_node/impl/base_intent_node.py delete mode 100644 apps/application/flow/step_node/intent_node/impl/prompt_template.py delete mode 100644 apps/application/flow/step_node/knowledge_write_node/__init__.py delete mode 100644 apps/application/flow/step_node/knowledge_write_node/i_knowledge_write_node.py delete mode 100644 apps/application/flow/step_node/knowledge_write_node/impl/__init__.py delete mode 100644 apps/application/flow/step_node/knowledge_write_node/impl/base_knowledge_write_node.py delete mode 100644 apps/application/flow/step_node/loop_break_node/__init__.py delete mode 100644 apps/application/flow/step_node/loop_break_node/i_loop_break_node.py delete mode 100644 apps/application/flow/step_node/loop_break_node/impl/__init__.py delete mode 100644 apps/application/flow/step_node/loop_break_node/impl/base_loop_break_node.py delete mode 100644 apps/application/flow/step_node/loop_continue_node/__init__.py delete mode 100644 apps/application/flow/step_node/loop_continue_node/i_loop_continue_node.py delete mode 100644 apps/application/flow/step_node/loop_continue_node/impl/__init__.py delete mode 100644 apps/application/flow/step_node/loop_continue_node/impl/base_loop_continue_node.py delete mode 100644 apps/application/flow/step_node/loop_node/__init__.py delete mode 100644 apps/application/flow/step_node/loop_node/i_loop_node.py delete mode 100644 apps/application/flow/step_node/loop_node/impl/__init__.py delete mode 100644 apps/application/flow/step_node/loop_node/impl/base_loop_node.py delete mode 100644 apps/application/flow/step_node/loop_start_node/__init__.py delete mode 100644 apps/application/flow/step_node/loop_start_node/i_loop_start_node.py delete mode 100644 apps/application/flow/step_node/loop_start_node/impl/__init__.py delete mode 100644 apps/application/flow/step_node/loop_start_node/impl/base_start_node.py delete mode 100644 apps/application/flow/step_node/mcp_node/__init__.py delete mode 100644 apps/application/flow/step_node/mcp_node/i_mcp_node.py delete mode 100644 apps/application/flow/step_node/mcp_node/impl/__init__.py delete mode 100644 apps/application/flow/step_node/mcp_node/impl/base_mcp_node.py delete mode 100644 apps/application/flow/step_node/parameter_extraction_node/__init__.py delete mode 100644 apps/application/flow/step_node/parameter_extraction_node/i_parameter_extraction_node.py delete mode 100644 apps/application/flow/step_node/parameter_extraction_node/impl/__init__.py delete mode 100644 apps/application/flow/step_node/parameter_extraction_node/impl/base_parameter_extraction_node.py delete mode 100644 apps/application/flow/step_node/question_node/__init__.py delete mode 100644 apps/application/flow/step_node/question_node/i_question_node.py delete mode 100644 apps/application/flow/step_node/question_node/impl/__init__.py delete mode 100644 apps/application/flow/step_node/question_node/impl/base_question_node.py delete mode 100644 apps/application/flow/step_node/reranker_node/__init__.py delete mode 100644 apps/application/flow/step_node/reranker_node/i_reranker_node.py delete mode 100644 apps/application/flow/step_node/reranker_node/impl/__init__.py delete mode 100644 apps/application/flow/step_node/reranker_node/impl/base_reranker_node.py delete mode 100644 apps/application/flow/step_node/search_document_node/__init__.py delete mode 100644 apps/application/flow/step_node/search_document_node/i_search_document_node.py delete mode 100644 apps/application/flow/step_node/search_document_node/impl/__init__.py delete mode 100644 apps/application/flow/step_node/search_document_node/impl/base_search_document_node.py delete mode 100644 apps/application/flow/step_node/search_knowledge_node/__init__.py delete mode 100644 apps/application/flow/step_node/search_knowledge_node/i_search_knowledge_node.py delete mode 100644 apps/application/flow/step_node/search_knowledge_node/impl/__init__.py delete mode 100644 apps/application/flow/step_node/search_knowledge_node/impl/base_search_knowledge_node.py delete mode 100644 apps/application/flow/step_node/speech_to_text_step_node/__init__.py delete mode 100644 apps/application/flow/step_node/speech_to_text_step_node/i_speech_to_text_node.py delete mode 100644 apps/application/flow/step_node/speech_to_text_step_node/impl/__init__.py delete mode 100644 apps/application/flow/step_node/speech_to_text_step_node/impl/base_speech_to_text_node.py delete mode 100644 apps/application/flow/step_node/start_node/__init__.py delete mode 100644 apps/application/flow/step_node/start_node/i_start_node.py delete mode 100644 apps/application/flow/step_node/start_node/impl/__init__.py delete mode 100644 apps/application/flow/step_node/start_node/impl/base_start_node.py delete mode 100644 apps/application/flow/step_node/text_to_speech_step_node/__init__.py delete mode 100644 apps/application/flow/step_node/text_to_speech_step_node/i_text_to_speech_node.py delete mode 100644 apps/application/flow/step_node/text_to_speech_step_node/impl/__init__.py delete mode 100644 apps/application/flow/step_node/text_to_speech_step_node/impl/base_text_to_speech_node.py delete mode 100644 apps/application/flow/step_node/text_to_video_step_node/__init__.py delete mode 100644 apps/application/flow/step_node/text_to_video_step_node/i_text_to_video_node.py delete mode 100644 apps/application/flow/step_node/text_to_video_step_node/impl/__init__.py delete mode 100644 apps/application/flow/step_node/text_to_video_step_node/impl/base_text_to_video_node.py delete mode 100644 apps/application/flow/step_node/tool_lib_node/__init__.py delete mode 100644 apps/application/flow/step_node/tool_lib_node/i_tool_lib_node.py delete mode 100644 apps/application/flow/step_node/tool_lib_node/impl/__init__.py delete mode 100644 apps/application/flow/step_node/tool_lib_node/impl/base_tool_lib_node.py delete mode 100644 apps/application/flow/step_node/tool_node/__init__.py delete mode 100644 apps/application/flow/step_node/tool_node/i_tool_node.py delete mode 100644 apps/application/flow/step_node/tool_node/impl/__init__.py delete mode 100644 apps/application/flow/step_node/tool_node/impl/base_tool_node.py delete mode 100644 apps/application/flow/step_node/tool_start_node/__init__.py delete mode 100644 apps/application/flow/step_node/tool_start_node/i_tool_start_node.py delete mode 100644 apps/application/flow/step_node/tool_start_node/impl/__init__.py delete mode 100644 apps/application/flow/step_node/tool_start_node/impl/base_tool_start_node.py delete mode 100644 apps/application/flow/step_node/tool_workflow_lib_node/__init__.py delete mode 100644 apps/application/flow/step_node/tool_workflow_lib_node/i_tool_workflow_lib_node.py delete mode 100644 apps/application/flow/step_node/tool_workflow_lib_node/impl/__init__.py delete mode 100644 apps/application/flow/step_node/tool_workflow_lib_node/impl/base_tool_workflow_lib_node.py delete mode 100644 apps/application/flow/step_node/variable_aggregation_node/__init__.py delete mode 100644 apps/application/flow/step_node/variable_aggregation_node/i_variable_aggregation_node.py delete mode 100644 apps/application/flow/step_node/variable_aggregation_node/impl/__init__.py delete mode 100644 apps/application/flow/step_node/variable_aggregation_node/impl/base_variable_aggregation_node.py delete mode 100644 apps/application/flow/step_node/variable_assign_node/__init__.py delete mode 100644 apps/application/flow/step_node/variable_assign_node/i_variable_assign_node.py delete mode 100644 apps/application/flow/step_node/variable_assign_node/impl/__init__.py delete mode 100644 apps/application/flow/step_node/variable_assign_node/impl/base_variable_assign_node.py delete mode 100644 apps/application/flow/step_node/variable_splitting_node/__init__.py delete mode 100644 apps/application/flow/step_node/variable_splitting_node/i_variable_splitting_node.py delete mode 100644 apps/application/flow/step_node/variable_splitting_node/impl/__init__.py delete mode 100644 apps/application/flow/step_node/variable_splitting_node/impl/base_variable_splitting_node.py delete mode 100644 apps/application/flow/step_node/video_understand_step_node/__init__.py delete mode 100644 apps/application/flow/step_node/video_understand_step_node/i_video_understand_node.py delete mode 100644 apps/application/flow/step_node/video_understand_step_node/impl/__init__.py delete mode 100644 apps/application/flow/step_node/video_understand_step_node/impl/base_video_understand_node.py delete mode 100644 apps/application/flow/tool_loop_workflow_manage.py delete mode 100644 apps/application/flow/tool_workflow_manage.py delete mode 100644 apps/application/flow/tools.py delete mode 100644 apps/application/flow/workflow_manage.py rename apps/application/{flow => workflow}/backend/sandbox_mcp.py (100%) diff --git a/apps/application/flow/__init__.py b/apps/application/flow/__init__.py deleted file mode 100644 index 328e8f8ec5f..00000000000 --- a/apps/application/flow/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： __init__.py.py - @date：2024/6/7 14:43 - @desc: -""" diff --git a/apps/application/flow/backend/__init__.py b/apps/application/flow/backend/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/apps/application/flow/backend/sandbox_shell.py b/apps/application/flow/backend/sandbox_shell.py deleted file mode 100644 index 7dac1f90376..00000000000 --- a/apps/application/flow/backend/sandbox_shell.py +++ /dev/null @@ -1,311 +0,0 @@ -import getpass -import os -import re -import shlex - -from deepagents.backends import LocalShellBackend -from deepagents.backends.protocol import ExecuteResponse - -from common.utils.logger import maxkb_logger -from maxkb.const import CONFIG - -_enable_sandbox = bool(int(CONFIG.get("SANDBOX", 1))) -_run_user = "sandbox" if _enable_sandbox else getpass.getuser() -_sandbox_python_sys_path = CONFIG.get_sandbox_python_package_paths().replace(",", ":") - - -class SandboxShellBackend(LocalShellBackend): - def __init__(self, root_dir: str, **kwargs): - if "env" not in kwargs and not kwargs.get("inherit_env", False): - env = os.environ.copy() - python_path = env.get("PYTHONPATH", "") - - # 将 sandbox Python 包路径分解为列表，检查每个路径是否已存在 - existing_paths = set(python_path.split(os.pathsep)) - sandbox_paths = _sandbox_python_sys_path.split(os.pathsep) if _sandbox_python_sys_path else [] - new_paths = [p for p in sandbox_paths if p and p not in existing_paths] - - if new_paths: - env["PYTHONPATH"] = ( - f"{os.pathsep.join(new_paths)}{os.pathsep}{python_path}" - if python_path - else os.pathsep.join(new_paths) - ) - - kwargs["env"] = env - super().__init__(root_dir=root_dir, **kwargs) - - def _translate_virtual_paths(self, command: str) -> str: - """Translate virtual absolute paths in the command to real filesystem paths. - - In virtual_mode=True, file tools (ls, glob, read_file) return virtual absolute - paths like /skills/foo.py which map to {root_dir}/skills/foo.py. But execute() - runs a real shell where /skills/foo.py does not exist. This method replaces - any path token that exists under root_dir with its real path, while leaving - genuine system paths (e.g. /usr/bin/python3) untouched. - """ - root = str(self.cwd) - - def translate(m: re.Match) -> str: - virtual_path = m.group(0) - real_path = root + virtual_path - return real_path if os.path.lexists(real_path) else virtual_path - - # Match absolute-path-like tokens: / followed by a non-whitespace sequence - # that isn't clearly a flag (e.g. avoid matching -/something). - # Only translate when virtual_mode is active. - return re.sub(r'(?<:,]*', translate, command) - - def _consume_group(self, command: str, start_index: int) -> tuple[str, int]: - current = [] - in_single_quote = False - in_double_quote = False - in_backticks = False - escaped = False - substitution_depth = 0 - group_depth = 1 - index = start_index + 1 - - while index < len(command): - char = command[index] - - if escaped: - current.append(char) - escaped = False - index += 1 - continue - - if char == "\\" and not in_single_quote: - current.append(char) - escaped = True - index += 1 - continue - - if char == "`" and not in_single_quote: - in_backticks = not in_backticks - current.append(char) - index += 1 - continue - - if in_backticks: - current.append(char) - index += 1 - continue - - if char == "'" and not in_double_quote: - in_single_quote = not in_single_quote - current.append(char) - index += 1 - continue - - if char == '"' and not in_single_quote: - in_double_quote = not in_double_quote - current.append(char) - index += 1 - continue - - if in_single_quote or in_double_quote: - current.append(char) - index += 1 - continue - - if command.startswith("$(", index): - substitution_depth += 1 - current.append("$(") - index += 2 - continue - - if substitution_depth: - if char == ")": - substitution_depth -= 1 - current.append(char) - index += 1 - continue - - if char == "(": - group_depth += 1 - current.append(char) - index += 1 - continue - - if char == ")": - group_depth -= 1 - if group_depth == 0: - return "".join(current).strip(), index + 1 - current.append(char) - index += 1 - continue - - current.append(char) - index += 1 - - raise ValueError("unclosed command group") - - def _append_pending_command_part(self, parts: list[str | tuple[str, str]], current: list[str]) -> None: - part = "".join(current).strip() - if part: - parts.append(part) - return - - if not parts: - parts.append("") - return - - last_part = parts[-1] - if isinstance(last_part, str) and last_part in {";", "&&", "||", "|", "&"}: - parts.append("") - - def _split_shell_command_list(self, command: str) -> list[str | tuple[str, str]]: - parts = [] - current = [] - in_single_quote = False - in_double_quote = False - in_backticks = False - escaped = False - substitution_depth = 0 - index = 0 - - while index < len(command): - char = command[index] - - if escaped: - current.append(char) - escaped = False - index += 1 - continue - - if char == "\\" and not in_single_quote: - current.append(char) - escaped = True - index += 1 - continue - - if char == "`" and not in_single_quote: - in_backticks = not in_backticks - current.append(char) - index += 1 - continue - - if in_backticks: - current.append(char) - index += 1 - continue - - if char == "'" and not in_double_quote: - in_single_quote = not in_single_quote - current.append(char) - index += 1 - continue - - if char == '"' and not in_single_quote: - in_double_quote = not in_double_quote - current.append(char) - index += 1 - continue - - if not in_single_quote and not in_double_quote: - if command.startswith("$(", index): - substitution_depth += 1 - current.append("$(") - index += 2 - continue - - if substitution_depth: - if char == ")": - substitution_depth -= 1 - current.append(char) - index += 1 - continue - - if char == "(" and not "".join(current).strip(): - group_content, index = self._consume_group(command, index) - parts.append(("group", group_content)) - current = [] - continue - - if command.startswith("&&", index) or command.startswith("||", index): - self._append_pending_command_part(parts, current) - parts.append(command[index : index + 2]) - current = [] - index += 2 - continue - - if char in {";", "|", "&"}: - self._append_pending_command_part(parts, current) - parts.append(char) - current = [] - index += 1 - continue - - if char == "\n": - self._append_pending_command_part(parts, current) - parts.append(";") - current = [] - index += 1 - continue - - current.append(char) - index += 1 - - self._append_pending_command_part(parts, current) - return parts - - def _build_sandbox_command(self, command: str) -> str: - prefix = ( - "env -i LD_PRELOAD=/opt/maxkb-app/sandbox/lib/sandbox.so " - f'PATH="${{PATH}}" PYTHONPATH="${{PYTHONPATH}}" gosu {_run_user} ' - ) - parts = self._split_shell_command_list(command) - sandboxed_parts = [] - expect_command = True - - for part in parts: - if expect_command: - if isinstance(part, tuple): - group_kind, group_content = part - if group_kind != "group": - raise ValueError(f"unsupported command part: {group_kind}") - if not group_content: - raise ValueError("empty command group") - sandboxed_parts.append(f"( {self._build_sandbox_command(group_content)} )") - elif not part: - raise ValueError("empty command") - else: - tokens = shlex.split(part) - if not tokens: - raise ValueError("empty command") - sandboxed_parts.append(prefix + " ".join(shlex.quote(token) for token in tokens)) - else: - if part not in {";", "&&", "||", "|", "&"}: - raise ValueError(f"unsupported shell operator: {part}") - sandboxed_parts.append(part) - - expect_command = not expect_command - - if expect_command: - raise ValueError("command cannot end with a shell operator") - - return " ".join(sandboxed_parts) - - def execute( - self, - command: str, - *, - timeout: int | None = None, - ) -> ExecuteResponse: - if self.virtual_mode: - command = self._translate_virtual_paths(command) - - if _enable_sandbox: - # 用 runuser 在子进程里切换用户，父进程凭据保持不变， - # 避免父进程 ruid/euid 不一致导致 execve 报 Permission denied - try: - # 将命令列表拆成多个简单命令，并分别在 sandbox 用户下执行。 - # 每个简单命令仍按 argv 重新 quote，避免 $()、反引号等在父 shell 中展开。 - command = self._build_sandbox_command(command) - except ValueError as e: - return ExecuteResponse(output=f"Invalid command: {e}", exit_code=1) - # command = f"runuser -u {_run_user} -- env -i PATH=${{PATH}} {command}" - - maxkb_logger.debug(f"Executing command in sandbox: {command}") - return super().execute(command=command, timeout=timeout) diff --git a/apps/application/flow/common.py b/apps/application/flow/common.py deleted file mode 100644 index d7520cf690c..00000000000 --- a/apps/application/flow/common.py +++ /dev/null @@ -1,284 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎 - @file： common.py - @date：2024/12/11 17:57 - @desc: -""" -from enum import Enum -from typing import List, Dict - -from django.db.models import QuerySet -from django.utils.translation import gettext as _ -from rest_framework.exceptions import ErrorDetail, ValidationError - -from common.exception.app_exception import AppApiException -from common.utils.common import group_by -from models_provider.models import Model -from models_provider.tools import get_model_credential -from tools.models.tool import Tool - -end_nodes = ['ai-chat-node', 'reply-node', 'function-node', 'function-lib-node', 'application-node', - 'image-understand-node', 'speech-to-text-node', 'text-to-speech-node', 'image-generate-node', - 'variable-assign-node'] - - -class Answer: - def __init__(self, content, view_type, runtime_node_id, chat_record_id, child_node, real_node_id, - reasoning_content): - self.view_type = view_type - self.content = content - self.reasoning_content = reasoning_content - self.runtime_node_id = runtime_node_id - self.chat_record_id = chat_record_id - self.child_node = child_node - self.real_node_id = real_node_id - - def to_dict(self): - return {'view_type': self.view_type, 'content': self.content, 'runtime_node_id': self.runtime_node_id, - 'chat_record_id': self.chat_record_id, - 'child_node': self.child_node, - 'reasoning_content': self.reasoning_content, - 'real_node_id': self.real_node_id} - - -class NodeChunk: - def __init__(self): - self.status = 0 - self.chunk_list = [] - - def add_chunk(self, chunk): - self.chunk_list.append(chunk) - - def end(self, chunk=None): - if chunk is not None: - self.add_chunk(chunk) - self.status = 200 - - def is_end(self): - return self.status == 200 - - -class Edge: - def __init__(self, _id: str, _type: str, sourceNodeId: str, targetNodeId: str, **keywords): - self.id = _id - self.type = _type - self.sourceNodeId = sourceNodeId - self.targetNodeId = targetNodeId - for keyword in keywords: - self.__setattr__(keyword, keywords.get(keyword)) - - -class Node: - def __init__(self, _id: str, _type: str, x: int, y: int, properties: dict, **kwargs): - self.id = _id - self.type = _type - self.x = x - self.y = y - self.properties = properties - for keyword in kwargs: - self.__setattr__(keyword, kwargs.get(keyword)) - - -class EdgeNode: - edge: Edge - node: Node - - def __init__(self, edge, node): - self.edge = edge - self.node = node - - -class WorkflowMode(Enum): - APPLICATION = "application" - - APPLICATION_LOOP = "application-loop" - - KNOWLEDGE = "knowledge" - - KNOWLEDGE_LOOP = "knowledge-loop" - - TOOL = "tool" - - TOOL_LOOP = "tool-loop" - - -class Workflow: - """ - 节点列表 - """ - nodes: List[Node] - """ - 线列表 - """ - edges: List[Edge] - """ - 节点id:node - """ - node_map: Dict[str, Node] - """ - 节点id:当前节点id上面的所有节点 - """ - up_node_map: Dict[str, List[EdgeNode]] - """ - 节点id:当前节点id下面的所有节点 - """ - next_node_map: Dict[str, List[EdgeNode]] - - workflow_mode: WorkflowMode - - def __init__(self, nodes: List[Node], edges: List[Edge], - workflow_mode: WorkflowMode = WorkflowMode.APPLICATION.value): - self.nodes = nodes - self.edges = edges - self.node_map = {node.id: node for node in nodes} - - self.up_node_map = {key: [EdgeNode(edge, self.node_map.get(edge.sourceNodeId)) for - edge in edges] for - key, edges in - group_by(edges, key=lambda edge: edge.targetNodeId).items()} - - self.next_node_map = {key: [EdgeNode(edge, self.node_map.get(edge.targetNodeId)) for edge in edges] for - key, edges in - group_by(edges, key=lambda edge: edge.sourceNodeId).items()} - self.workflow_mode = workflow_mode - - def get_node(self, node_id): - """ - 根据node_id 获取节点信息 - @param node_id: node_id - @return: 节点信息 - """ - return self.node_map.get(node_id) - - def get_up_edge_nodes(self, node_id) -> List[EdgeNode]: - """ - 根据节点id 获取当前连接前置节点和连线 - @param node_id: 节点id - @return: 节点连线列表 - """ - return self.up_node_map.get(node_id) - - def get_next_edge_nodes(self, node_id) -> List[EdgeNode]: - """ - 根据节点id 获取当前连接目标节点和连线 - @param node_id: 节点id - @return: 节点连线列表 - """ - return self.next_node_map.get(node_id) - - def get_up_nodes(self, node_id) -> List[Node]: - """ - 根据节点id 获取当前连接前置节点 - @param node_id: 节点id - @return: 节点列表 - """ - return [en.node for en in (self.up_node_map.get(node_id) or [])] - - def get_next_nodes(self, node_id) -> List[Node]: - """ - 根据节点id 获取当前连接目标节点 - @param node_id: 节点id - @return: 节点列表 - """ - return [en.node for en in self.next_node_map.get(node_id, [])] - - @staticmethod - def new_instance(flow_obj: Dict, workflow_mode: WorkflowMode = WorkflowMode.APPLICATION): - nodes = flow_obj.get('nodes') - edges = flow_obj.get('edges') - nodes = [Node(node.get('id'), node.get('type'), **node) - for node in nodes] - edges = [Edge(edge.get('id'), edge.get('type'), **edge) for edge in edges] - return Workflow(nodes, edges, workflow_mode) - - def get_start_node(self): - return self.get_node('start-node') - - def get_search_node(self): - return [node for node in self.nodes if node.type == 'search-dataset-node'] - - def is_valid(self): - """ - 校验工作流数据 - """ - self.is_valid_model_params() - self.is_valid_start_node() - self.is_valid_base_node() - self.is_valid_work_flow() - - def is_valid_node_params(self, node: Node): - from application.flow.step_node import get_node - get_node(node.type, self.workflow_mode)(node, None, None) - - def is_valid_node(self, node: Node): - self.is_valid_node_params(node) - if node.type == 'condition-node': - branch_list = node.properties.get('node_data').get('branch') - for branch in branch_list: - source_anchor_id = f"{node.id}_{branch.get('id')}_right" - edge_list = [edge for edge in self.edges if edge.sourceAnchorId == source_anchor_id] - if len(edge_list) == 0: - raise AppApiException(500, - _('The branch {branch} of the {node} node needs to be connected').format( - node=node.properties.get("stepName"), branch=branch.get("type"))) - - else: - edge_list = [edge for edge in self.edges if edge.sourceNodeId == node.id] - if len(edge_list) == 0 and not end_nodes.__contains__(node.type): - raise AppApiException(500, _("{node} Nodes cannot be considered as end nodes").format( - node=node.properties.get("stepName"))) - - def is_valid_work_flow(self, up_node=None): - if up_node is None: - up_node = self.get_start_node() - self.is_valid_node(up_node) - next_nodes = self.get_next_nodes(up_node) - for next_node in next_nodes: - self.is_valid_work_flow(next_node) - - def is_valid_start_node(self): - start_node_list = [node for node in self.nodes if node.id == 'start-node'] - if len(start_node_list) == 0: - raise AppApiException(500, _('The starting node is required')) - if len(start_node_list) > 1: - raise AppApiException(500, _('There can only be one starting node')) - - def is_valid_model_params(self): - node_list = [node for node in self.nodes if ( - node.type == 'ai-chat-node' or node.type == 'question-node' or node.type == 'parameter-extraction-node')] - for node in node_list: - if (node.properties.get('node_data', {}).get('model_id_type') or 'custom') == 'reference': - continue - model = QuerySet(Model).filter(id=node.properties.get('node_data', {}).get('model_id')).first() - if model is None: - raise ValidationError(ErrorDetail( - _('The node {node} model does not exist').format(node=node.properties.get("stepName")))) - credential = get_model_credential(model.provider, model.model_type, model.model_name) - model_params_setting = node.properties.get('node_data', {}).get('model_params_setting') - model_params_setting_form = credential.get_model_params_setting_form( - model.model_name) - if model_params_setting is None: - model_params_setting = model_params_setting_form.get_default_form_data() - node.properties.get('node_data', {})['model_params_setting'] = model_params_setting - if node.properties.get('status', 200) != 200: - raise ValidationError( - ErrorDetail(_("Node {node} is unavailable").format(node=node.properties.get("stepName")))) - node_list = [node for node in self.nodes if (node.type == 'function-lib-node')] - for node in node_list: - function_lib_id = node.properties.get('node_data', {}).get('function_lib_id') - if function_lib_id is None: - raise ValidationError(ErrorDetail( - _('The library ID of node {node} cannot be empty').format(node=node.properties.get("stepName")))) - f_lib = QuerySet(Tool).filter(id=function_lib_id).first() - if f_lib is None: - raise ValidationError(ErrorDetail(_("The function library for node {node} is not available").format( - node=node.properties.get("stepName")))) - - def is_valid_base_node(self): - base_node_list = [node for node in self.nodes if node.id == 'base-node'] - if len(base_node_list) == 0: - raise AppApiException(500, _('Basic information node is required')) - if len(base_node_list) > 1: - raise AppApiException(500, _('There can only be one basic information node')) diff --git a/apps/application/flow/compare/__init__.py b/apps/application/flow/compare/__init__.py deleted file mode 100644 index ce0c430e1ad..00000000000 --- a/apps/application/flow/compare/__init__.py +++ /dev/null @@ -1,83 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： __init__.py.py - @date：2024/6/7 14:43 - @desc: -""" -from typing import List - -from .contain_compare import ContainCompare -from .end_with import EndWithCompare -from .equal_compare import EqualCompare -from .ge_compare import GECompare -from .gt_compare import GTCompare -from .is_not_null_compare import IsNotNullCompare -from .is_not_true import IsNotTrueCompare -from .is_null_compare import IsNullCompare -from .is_true import IsTrueCompare -from .le_compare import LECompare -from .len_equal_compare import LenEqualCompare -from .len_ge_compare import LenGECompare -from .len_gt_compare import LenGTCompare -from .len_le_compare import LenLECompare -from .len_lt_compare import LenLTCompare -from .lt_compare import LTCompare -from .not_contain_compare import NotContainCompare -from .not_equal_compare import NotEqualCompare -from .regex_compare import RegexCompare -from .start_with import StartWithCompare -from .wildcard_compare import WildcardCompare - -_compare_handler_dict = { - 'is_null': IsNullCompare(), - 'is_not_null': IsNotNullCompare(), - 'contain': ContainCompare(), - 'not_contain': NotContainCompare(), - 'eq': EqualCompare(), - 'not_eq': NotEqualCompare(), - 'ge': GECompare(), - 'gt': GTCompare(), - 'le': LECompare(), - 'lt': LTCompare(), - 'len_eq': LenEqualCompare(), - 'len_ge': LenGECompare(), - 'len_gt': LenGTCompare(), - 'len_le': LenLECompare(), - 'len_lt': LenLTCompare(), - 'is_true': IsTrueCompare(), - 'is_not_true': IsNotTrueCompare(), - 'start_with': StartWithCompare(), - 'end_with': EndWithCompare(), - 'regex': RegexCompare(), - 'wildcard': WildcardCompare(), -} - - -def _compare(source_value, compare, target_value): - compare_handler = _compare_handler_dict.get(compare) - if compare_handler is None: - raise RuntimeError(f"Unknown compare handler '{compare}'") - return compare_handler.compare(source_value, compare, target_value) - - -def _assertion(workflow_manage, field_list: List[str], compare: str, value): - try: - value = workflow_manage.generate_prompt(value) - except Exception: - pass - field_value = None - try: - field_value = workflow_manage.get_reference_field(field_list[0], field_list[1:]) - except Exception: - pass - return _compare(field_value, compare, value) - - -def do_assertion(workflow_manage, condition, condition_list): - b = False if condition == 'and' else True - for row in condition_list: - if _assertion(workflow_manage, row.get('field'), row.get('compare'), row.get('value')) is b: - return b - return not b diff --git a/apps/application/flow/compare/compare.py b/apps/application/flow/compare/compare.py deleted file mode 100644 index 62eb4a7b910..00000000000 --- a/apps/application/flow/compare/compare.py +++ /dev/null @@ -1,15 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： compare.py - @date：2024/6/7 14:37 - @desc: -""" -from abc import abstractmethod - -class Compare: - - @abstractmethod - def compare(self, source_value, compare, target_value): - pass diff --git a/apps/application/flow/compare/contain_compare.py b/apps/application/flow/compare/contain_compare.py deleted file mode 100644 index cd50d543a3a..00000000000 --- a/apps/application/flow/compare/contain_compare.py +++ /dev/null @@ -1,25 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： contain_compare.py - @date：2024/6/11 10:02 - @desc: -""" -from .compare import Compare - - -class ContainCompare(Compare): - - def compare(self, source_value, compare, target_value): - target_value = str(target_value) - - if isinstance(source_value, str): - return target_value in source_value - elif isinstance(source_value, list): - for item in source_value: - if str(item) == target_value: - return True - return False - else: - return target_value in str(source_value) diff --git a/apps/application/flow/compare/end_with.py b/apps/application/flow/compare/end_with.py deleted file mode 100644 index eae7e3a8a15..00000000000 --- a/apps/application/flow/compare/end_with.py +++ /dev/null @@ -1,16 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎虎 - @file： start_with.py - @date：2025/10/20 10:37 - @desc: -""" -from .compare import Compare - - -class EndWithCompare(Compare): - - def compare(self, source_value, compare, target_value): - source_value = str(source_value) - return source_value.endswith(str(target_value)) diff --git a/apps/application/flow/compare/equal_compare.py b/apps/application/flow/compare/equal_compare.py deleted file mode 100644 index dad0cffa9fa..00000000000 --- a/apps/application/flow/compare/equal_compare.py +++ /dev/null @@ -1,15 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： equal_compare.py - @date：2024/6/7 14:44 - @desc: -""" -from .compare import Compare - - -class EqualCompare(Compare): - - def compare(self, source_value, compare, target_value): - return str(source_value) == str(target_value) diff --git a/apps/application/flow/compare/ge_compare.py b/apps/application/flow/compare/ge_compare.py deleted file mode 100644 index e1cf2e7aac0..00000000000 --- a/apps/application/flow/compare/ge_compare.py +++ /dev/null @@ -1,25 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： lt_compare.py - @date：2024/6/11 9:52 - @desc: 大于比较器 -""" -from .compare import Compare - - -class GECompare(Compare): - - def compare(self, source_value, compare, target_value): - if source_value is None: - return target_value is None - - try: - return float(source_value) >= float(target_value) - except Exception: - try: - return str(source_value) >= str(target_value) - except Exception: - pass - return False diff --git a/apps/application/flow/compare/gt_compare.py b/apps/application/flow/compare/gt_compare.py deleted file mode 100644 index fab86c4bc8c..00000000000 --- a/apps/application/flow/compare/gt_compare.py +++ /dev/null @@ -1,25 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： lt_compare.py - @date：2024/6/11 9:52 - @desc: 大于比较器 -""" -from .compare import Compare - - -class GTCompare(Compare): - - def compare(self, source_value, compare, target_value): - if source_value is None: - return False - - try: - return float(source_value) > float(target_value) - except Exception: - try: - return str(source_value) > str(target_value) - except Exception: - pass - return False diff --git a/apps/application/flow/compare/is_not_null_compare.py b/apps/application/flow/compare/is_not_null_compare.py deleted file mode 100644 index 37fd4a72ea4..00000000000 --- a/apps/application/flow/compare/is_not_null_compare.py +++ /dev/null @@ -1,18 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： is_not_null_compare.py - @date：2024/6/28 10:45 - @desc: -""" -from .compare import Compare - - -class IsNotNullCompare(Compare): - - def compare(self, source_value, compare, target_value): - try: - return source_value is not None and len(source_value) > 0 - except Exception: - return True diff --git a/apps/application/flow/compare/is_not_true.py b/apps/application/flow/compare/is_not_true.py deleted file mode 100644 index fabeec2cc41..00000000000 --- a/apps/application/flow/compare/is_not_true.py +++ /dev/null @@ -1,18 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎 - @file： is_not_true.py - @date：2025/4/7 13:44 - @desc: -""" -from .compare import Compare - - -class IsNotTrueCompare(Compare): - - def compare(self, source_value, compare, target_value): - try: - return source_value is False - except Exception: - return False diff --git a/apps/application/flow/compare/is_null_compare.py b/apps/application/flow/compare/is_null_compare.py deleted file mode 100644 index 240403ea605..00000000000 --- a/apps/application/flow/compare/is_null_compare.py +++ /dev/null @@ -1,18 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： is_null_compare.py - @date：2024/6/28 10:45 - @desc: -""" -from .compare import Compare - - -class IsNullCompare(Compare): - - def compare(self, source_value, compare, target_value): - try: - return source_value is None or len(source_value) == 0 - except Exception: - return False diff --git a/apps/application/flow/compare/is_true.py b/apps/application/flow/compare/is_true.py deleted file mode 100644 index 8cb4a45a2a5..00000000000 --- a/apps/application/flow/compare/is_true.py +++ /dev/null @@ -1,18 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎 - @file： IsTrue.py - @date：2025/4/7 13:38 - @desc: -""" -from .compare import Compare - - -class IsTrueCompare(Compare): - - def compare(self, source_value, compare, target_value): - try: - return source_value is True - except Exception: - return False diff --git a/apps/application/flow/compare/le_compare.py b/apps/application/flow/compare/le_compare.py deleted file mode 100644 index 0ebdb394857..00000000000 --- a/apps/application/flow/compare/le_compare.py +++ /dev/null @@ -1,25 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： lt_compare.py - @date：2024/6/11 9:52 - @desc: 小于比较器 -""" -from .compare import Compare - - -class LECompare(Compare): - - def compare(self, source_value, compare, target_value): - if source_value is None: - return target_value is None - - try: - return float(source_value) <= float(target_value) - except Exception: - try: - return str(source_value) <= str(target_value) - except Exception: - pass - return False diff --git a/apps/application/flow/compare/len_equal_compare.py b/apps/application/flow/compare/len_equal_compare.py deleted file mode 100644 index 98a5314a292..00000000000 --- a/apps/application/flow/compare/len_equal_compare.py +++ /dev/null @@ -1,18 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： equal_compare.py - @date：2024/6/7 14:44 - @desc: -""" -from .compare import Compare - - -class LenEqualCompare(Compare): - - def compare(self, source_value, compare, target_value): - try: - return len(source_value) == int(target_value) - except Exception as e: - return False diff --git a/apps/application/flow/compare/len_ge_compare.py b/apps/application/flow/compare/len_ge_compare.py deleted file mode 100644 index 06dd566cf24..00000000000 --- a/apps/application/flow/compare/len_ge_compare.py +++ /dev/null @@ -1,18 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： lt_compare.py - @date：2024/6/11 9:52 - @desc: 大于比较器 -""" -from .compare import Compare - - -class LenGECompare(Compare): - - def compare(self, source_value, compare, target_value): - try: - return len(source_value) >= int(target_value) - except Exception: - return False diff --git a/apps/application/flow/compare/len_gt_compare.py b/apps/application/flow/compare/len_gt_compare.py deleted file mode 100644 index fae2668e0ba..00000000000 --- a/apps/application/flow/compare/len_gt_compare.py +++ /dev/null @@ -1,18 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： lt_compare.py - @date：2024/6/11 9:52 - @desc: 大于比较器 -""" -from .compare import Compare - - -class LenGTCompare(Compare): - - def compare(self, source_value, compare, target_value): - try: - return len(source_value) > int(target_value) - except Exception: - return False diff --git a/apps/application/flow/compare/len_le_compare.py b/apps/application/flow/compare/len_le_compare.py deleted file mode 100644 index 41b9ee9f709..00000000000 --- a/apps/application/flow/compare/len_le_compare.py +++ /dev/null @@ -1,18 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： lt_compare.py - @date：2024/6/11 9:52 - @desc: 小于比较器 -""" -from .compare import Compare - - -class LenLECompare(Compare): - - def compare(self, source_value, compare, target_value): - try: - return len(source_value) <= int(target_value) - except Exception: - return False diff --git a/apps/application/flow/compare/len_lt_compare.py b/apps/application/flow/compare/len_lt_compare.py deleted file mode 100644 index 4a9b11654ca..00000000000 --- a/apps/application/flow/compare/len_lt_compare.py +++ /dev/null @@ -1,18 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： lt_compare.py - @date：2024/6/11 9:52 - @desc: 小于比较器 -""" -from .compare import Compare - - -class LenLTCompare(Compare): - - def compare(self, source_value, compare, target_value): - try: - return len(source_value) < int(target_value) - except Exception: - return False diff --git a/apps/application/flow/compare/lt_compare.py b/apps/application/flow/compare/lt_compare.py deleted file mode 100644 index ecf8d549bfd..00000000000 --- a/apps/application/flow/compare/lt_compare.py +++ /dev/null @@ -1,25 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： lt_compare.py - @date：2024/6/11 9:52 - @desc: 小于比较器 -""" -from .compare import Compare - - -class LTCompare(Compare): - - def compare(self, source_value, compare, target_value): - if source_value is None: - return False - - try: - return float(source_value) < float(target_value) - except Exception: - try: - return str(source_value) < str(target_value) - except Exception: - pass - return False diff --git a/apps/application/flow/compare/not_contain_compare.py b/apps/application/flow/compare/not_contain_compare.py deleted file mode 100644 index 99194e70364..00000000000 --- a/apps/application/flow/compare/not_contain_compare.py +++ /dev/null @@ -1,25 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： contain_compare.py - @date：2024/6/11 10:02 - @desc: -""" -from .compare import Compare - - -class NotContainCompare(Compare): - - def compare(self, source_value, compare, target_value): - target_value = str(target_value) - - if isinstance(source_value, str): - return target_value not in source_value - elif isinstance(source_value, list): - for item in source_value: - if str(item) == target_value: - return False - return True - else: - return target_value not in str(source_value) diff --git a/apps/application/flow/compare/not_equal_compare.py b/apps/application/flow/compare/not_equal_compare.py deleted file mode 100644 index f53057ebfa9..00000000000 --- a/apps/application/flow/compare/not_equal_compare.py +++ /dev/null @@ -1,15 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：wangliang181230 - @file： not_equal_compare.py - @date：2026/3/17 9:41 - @desc: -""" -from .compare import Compare - - -class NotEqualCompare(Compare): - - def compare(self, source_value, compare, target_value): - return str(source_value) != str(target_value) diff --git a/apps/application/flow/compare/regex_compare.py b/apps/application/flow/compare/regex_compare.py deleted file mode 100644 index 613300e6589..00000000000 --- a/apps/application/flow/compare/regex_compare.py +++ /dev/null @@ -1,35 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：wangliang181230 - @file： regex_compare.py - @date：2026/3/30 12:11 - @desc: -""" -import re - -from .compare import Compare -from common.cache.mem_cache import MemCache - -match_cache = MemCache('regex', { - 'TIMEOUT': 3600, # 缓存有效期为 1 小时 - 'OPTIONS': { - 'MAX_ENTRIES': 500, # 最多缓存 500 个条目 - 'CULL_FREQUENCY': 10, # 达到上限时，删除约 1/10 的缓存 - }, -}) - - -def compile_and_cache(regex): - match = match_cache.get(regex) - if not match: - match = re.compile(regex).fullmatch - match_cache.set(regex, match) - return match - - -class RegexCompare(Compare): - - def compare(self, source_value, compare, target_value): - match = compile_and_cache(str(target_value)) - return bool(match(str(source_value))) diff --git a/apps/application/flow/compare/start_with.py b/apps/application/flow/compare/start_with.py deleted file mode 100644 index 054ea9bd6cb..00000000000 --- a/apps/application/flow/compare/start_with.py +++ /dev/null @@ -1,16 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎虎 - @file： start_with.py - @date：2025/10/20 10:37 - @desc: -""" -from .compare import Compare - - -class StartWithCompare(Compare): - - def compare(self, source_value, compare, target_value): - source_value = str(source_value) - return source_value.startswith(str(target_value)) diff --git a/apps/application/flow/compare/wildcard_compare.py b/apps/application/flow/compare/wildcard_compare.py deleted file mode 100644 index 43c903a9360..00000000000 --- a/apps/application/flow/compare/wildcard_compare.py +++ /dev/null @@ -1,38 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：wangliang181230 - @file： wildcard_compare.py - @date：2026/3/30 12:11 - @desc: -""" -import fnmatch -import re - -from .compare import Compare -from common.cache.mem_cache import MemCache - - -match_cache = MemCache('wildcard_to_regex', { - 'TIMEOUT': 3600, # 缓存有效期为 1 小时 - 'OPTIONS': { - 'MAX_ENTRIES': 500, # 最多缓存 500 个条目 - 'CULL_FREQUENCY': 10, # 达到上限时，删除约 1/10 的缓存 - }, -}) - - -def translate_and_compile_and_cache(wildcard): - match = match_cache.get(wildcard) - if not match: - regex = fnmatch.translate(wildcard) - match = re.compile(regex).match - match_cache.set(wildcard, match) - return match - -class WildcardCompare(Compare): - - def compare(self, source_value, compare, target_value): - # 转成正则，性能更高 - match = translate_and_compile_and_cache(str(target_value)) - return bool(match(str(source_value))) diff --git a/apps/application/flow/default_workflow.json b/apps/application/flow/default_workflow.json deleted file mode 100644 index 48ac23c4dc6..00000000000 --- a/apps/application/flow/default_workflow.json +++ /dev/null @@ -1,451 +0,0 @@ -{ - "nodes": [ - { - "id": "base-node", - "type": "base-node", - "x": 360, - "y": 2810, - "properties": { - "config": { - - }, - "height": 825.6, - "stepName": "基本信息", - "node_data": { - "desc": "", - "name": "maxkbapplication", - "prologue": "您好，我是 MaxKB 小助手，您可以向我提出 MaxKB 使用问题。\n- MaxKB 主要功能有什么？\n- MaxKB 支持哪些大语言模型？\n- MaxKB 支持哪些文档类型？" - }, - "input_field_list": [ - - ] - } - }, - { - "id": "start-node", - "type": "start-node", - "x": 430, - "y": 3660, - "properties": { - "config": { - "fields": [ - { - "label": "用户问题", - "value": "question" - } - ], - "globalFields": [ - { - "label": "当前时间", - "value": "time" - } - ] - }, - "fields": [ - { - "label": "用户问题", - "value": "question" - } - ], - "height": 276, - "stepName": "开始", - "globalFields": [ - { - "label": "当前时间", - "value": "time" - } - ] - } - }, - { - "id": "b931efe5-5b66-46e0-ae3b-0160cb18eeb5", - "type": "search-dataset-node", - "x": 840, - "y": 3210, - "properties": { - "config": { - "fields": [ - { - "label": "检索结果的分段列表", - "value": "paragraph_list" - }, - { - "label": "满足直接回答的分段列表", - "value": "is_hit_handling_method_list" - }, - { - "label": "检索结果", - "value": "data" - }, - { - "label": "满足直接回答的分段内容", - "value": "directly_return" - } - ] - }, - "height": 794, - "stepName": "知识库检索", - "node_data": { - "dataset_id_list": [ - - ], - "dataset_setting": { - "top_n": 3, - "similarity": 0.6, - "search_mode": "embedding", - "max_paragraph_char_number": 5000 - }, - "question_reference_address": [ - "start-node", - "question" - ], - "source_dataset_id_list": [ - - ] - } - } - }, - { - "id": "fc60863a-dec2-4854-9e5a-7a44b7187a2b", - "type": "condition-node", - "x": 1490, - "y": 3210, - "properties": { - "width": 600, - "config": { - "fields": [ - { - "label": "分支名称", - "value": "branch_name" - } - ] - }, - "height": 543.675, - "stepName": "判断器", - "node_data": { - "branch": [ - { - "id": "1009", - "type": "IF", - "condition": "and", - "conditions": [ - { - "field": [ - "b931efe5-5b66-46e0-ae3b-0160cb18eeb5", - "is_hit_handling_method_list" - ], - "value": "1", - "compare": "len_ge" - } - ] - }, - { - "id": "4908", - "type": "ELSE IF 1", - "condition": "and", - "conditions": [ - { - "field": [ - "b931efe5-5b66-46e0-ae3b-0160cb18eeb5", - "paragraph_list" - ], - "value": "1", - "compare": "len_ge" - } - ] - }, - { - "id": "161", - "type": "ELSE", - "condition": "and", - "conditions": [ - - ] - } - ] - }, - "branch_condition_list": [ - { - "index": 0, - "height": 121.225, - "id": "1009" - }, - { - "index": 1, - "height": 121.225, - "id": "4908" - }, - { - "index": 2, - "height": 44, - "id": "161" - } - ] - } - }, - { - "id": "4ffe1086-25df-4c85-b168-979b5bbf0a26", - "type": "reply-node", - "x": 2170, - "y": 2480, - "properties": { - "config": { - "fields": [ - { - "label": "内容", - "value": "answer" - } - ] - }, - "height": 378, - "stepName": "指定回复", - "node_data": { - "fields": [ - "b931efe5-5b66-46e0-ae3b-0160cb18eeb5", - "directly_return" - ], - "content": "", - "reply_type": "referencing", - "is_result": true - } - } - }, - { - "id": "f1f1ee18-5a02-46f6-b4e6-226253cdffbb", - "type": "ai-chat-node", - "x": 2160, - "y": 3200, - "properties": { - "config": { - "fields": [ - { - "label": "AI 回答内容", - "value": "answer" - } - ] - }, - "height": 763, - "stepName": "AI 对话", - "node_data": { - "prompt": "已知信息：\n{{知识库检索.data}}\n问题：\n{{开始.question}}", - "system": "", - "model_id": "", - "dialogue_number": 0, - "is_result": true - } - } - }, - { - "id": "309d0eef-c597-46b5-8d51-b9a28aaef4c7", - "type": "ai-chat-node", - "x": 2160, - "y": 3970, - "properties": { - "config": { - "fields": [ - { - "label": "AI 回答内容", - "value": "answer" - } - ] - }, - "height": 763, - "stepName": "AI 对话1", - "node_data": { - "prompt": "{{开始.question}}", - "system": "", - "model_id": "", - "dialogue_number": 0, - "is_result": true - } - } - } - ], - "edges": [ - { - "id": "7d0f166f-c472-41b2-b9a2-c294f4c83d73", - "type": "app-edge", - "sourceNodeId": "start-node", - "targetNodeId": "b931efe5-5b66-46e0-ae3b-0160cb18eeb5", - "startPoint": { - "x": 590, - "y": 3660 - }, - "endPoint": { - "x": 680, - "y": 3210 - }, - "properties": { - - }, - "pointsList": [ - { - "x": 590, - "y": 3660 - }, - { - "x": 700, - "y": 3660 - }, - { - "x": 570, - "y": 3210 - }, - { - "x": 680, - "y": 3210 - } - ], - "sourceAnchorId": "start-node_right", - "targetAnchorId": "b931efe5-5b66-46e0-ae3b-0160cb18eeb5_left" - }, - { - "id": "35cb86dd-f328-429e-a973-12fd7218b696", - "type": "app-edge", - "sourceNodeId": "b931efe5-5b66-46e0-ae3b-0160cb18eeb5", - "targetNodeId": "fc60863a-dec2-4854-9e5a-7a44b7187a2b", - "startPoint": { - "x": 1000, - "y": 3210 - }, - "endPoint": { - "x": 1200, - "y": 3210 - }, - "properties": { - - }, - "pointsList": [ - { - "x": 1000, - "y": 3210 - }, - { - "x": 1110, - "y": 3210 - }, - { - "x": 1090, - "y": 3210 - }, - { - "x": 1200, - "y": 3210 - } - ], - "sourceAnchorId": "b931efe5-5b66-46e0-ae3b-0160cb18eeb5_right", - "targetAnchorId": "fc60863a-dec2-4854-9e5a-7a44b7187a2b_left" - }, - { - "id": "e8f6cfe6-7e48-41cd-abd3-abfb5304d0d8", - "type": "app-edge", - "sourceNodeId": "fc60863a-dec2-4854-9e5a-7a44b7187a2b", - "targetNodeId": "4ffe1086-25df-4c85-b168-979b5bbf0a26", - "startPoint": { - "x": 1780, - "y": 3073.775 - }, - "endPoint": { - "x": 2010, - "y": 2480 - }, - "properties": { - - }, - "pointsList": [ - { - "x": 1780, - "y": 3073.775 - }, - { - "x": 1890, - "y": 3073.775 - }, - { - "x": 1900, - "y": 2480 - }, - { - "x": 2010, - "y": 2480 - } - ], - "sourceAnchorId": "fc60863a-dec2-4854-9e5a-7a44b7187a2b_1009_right", - "targetAnchorId": "4ffe1086-25df-4c85-b168-979b5bbf0a26_left" - }, - { - "id": "994ff325-6f7a-4ebc-b61b-10e15519d6d2", - "type": "app-edge", - "sourceNodeId": "fc60863a-dec2-4854-9e5a-7a44b7187a2b", - "targetNodeId": "f1f1ee18-5a02-46f6-b4e6-226253cdffbb", - "startPoint": { - "x": 1780, - "y": 3203 - }, - "endPoint": { - "x": 2000, - "y": 3200 - }, - "properties": { - - }, - "pointsList": [ - { - "x": 1780, - "y": 3203 - }, - { - "x": 1890, - "y": 3203 - }, - { - "x": 1890, - "y": 3200 - }, - { - "x": 2000, - "y": 3200 - } - ], - "sourceAnchorId": "fc60863a-dec2-4854-9e5a-7a44b7187a2b_4908_right", - "targetAnchorId": "f1f1ee18-5a02-46f6-b4e6-226253cdffbb_left" - }, - { - "id": "19270caf-bb9f-4ba7-9bf8-200aa70fecd5", - "type": "app-edge", - "sourceNodeId": "fc60863a-dec2-4854-9e5a-7a44b7187a2b", - "targetNodeId": "309d0eef-c597-46b5-8d51-b9a28aaef4c7", - "startPoint": { - "x": 1780, - "y": 3293.6124999999997 - }, - "endPoint": { - "x": 2000, - "y": 3970 - }, - "properties": { - - }, - "pointsList": [ - { - "x": 1780, - "y": 3293.6124999999997 - }, - { - "x": 1890, - "y": 3293.6124999999997 - }, - { - "x": 1890, - "y": 3970 - }, - { - "x": 2000, - "y": 3970 - } - ], - "sourceAnchorId": "fc60863a-dec2-4854-9e5a-7a44b7187a2b_161_right", - "targetAnchorId": "309d0eef-c597-46b5-8d51-b9a28aaef4c7_left" - } - ] -} \ No newline at end of file diff --git a/apps/application/flow/default_workflow_en.json b/apps/application/flow/default_workflow_en.json deleted file mode 100644 index 17c397306b9..00000000000 --- a/apps/application/flow/default_workflow_en.json +++ /dev/null @@ -1,451 +0,0 @@ -{ - "nodes": [ - { - "id": "base-node", - "type": "base-node", - "x": 360, - "y": 2810, - "properties": { - "config": { - - }, - "height": 825.6, - "stepName": "Base", - "node_data": { - "desc": "", - "name": "maxkbapplication", - "prologue": "Hello, I am the MaxKB assistant. You can ask me about MaxKB usage issues.\n-What are the main functions of MaxKB?\n-What major language models does MaxKB support?\n-What document types does MaxKB support?" - }, - "input_field_list": [ - - ] - } - }, - { - "id": "start-node", - "type": "start-node", - "x": 430, - "y": 3660, - "properties": { - "config": { - "fields": [ - { - "label": "User Question", - "value": "question" - } - ], - "globalFields": [ - { - "label": "Current Time", - "value": "time" - } - ] - }, - "fields": [ - { - "label": "User Question", - "value": "question" - } - ], - "height": 276, - "stepName": "Start", - "globalFields": [ - { - "label": "Current Time", - "value": "time" - } - ] - } - }, - { - "id": "b931efe5-5b66-46e0-ae3b-0160cb18eeb5", - "type": "search-dataset-node", - "x": 840, - "y": 3210, - "properties": { - "config": { - "fields": [ - { - "label": "List of Retrieved Paragraphs", - "value": "paragraph_list" - }, - { - "label": "List of Paragraphs Satisfying Direct Answer", - "value": "is_hit_handling_method_list" - }, - { - "label": "Search Results", - "value": "data" - }, - { - "label": "Content of Paragraphs Satisfying Direct Answer", - "value": "directly_return" - } - ] - }, - "height": 794, - "stepName": "Knowledge Search", - "node_data": { - "dataset_id_list": [ - - ], - "dataset_setting": { - "top_n": 3, - "similarity": 0.6, - "search_mode": "embedding", - "max_paragraph_char_number": 5000 - }, - "question_reference_address": [ - "start-node", - "question" - ], - "source_dataset_id_list": [ - - ] - } - } - }, - { - "id": "fc60863a-dec2-4854-9e5a-7a44b7187a2b", - "type": "condition-node", - "x": 1490, - "y": 3210, - "properties": { - "width": 600, - "config": { - "fields": [ - { - "label": "Branch Name", - "value": "branch_name" - } - ] - }, - "height": 543.675, - "stepName": "Conditional Branch", - "node_data": { - "branch": [ - { - "id": "1009", - "type": "IF", - "condition": "and", - "conditions": [ - { - "field": [ - "b931efe5-5b66-46e0-ae3b-0160cb18eeb5", - "is_hit_handling_method_list" - ], - "value": "1", - "compare": "len_ge" - } - ] - }, - { - "id": "4908", - "type": "ELSE IF 1", - "condition": "and", - "conditions": [ - { - "field": [ - "b931efe5-5b66-46e0-ae3b-0160cb18eeb5", - "paragraph_list" - ], - "value": "1", - "compare": "len_ge" - } - ] - }, - { - "id": "161", - "type": "ELSE", - "condition": "and", - "conditions": [ - - ] - } - ] - }, - "branch_condition_list": [ - { - "index": 0, - "height": 121.225, - "id": "1009" - }, - { - "index": 1, - "height": 121.225, - "id": "4908" - }, - { - "index": 2, - "height": 44, - "id": "161" - } - ] - } - }, - { - "id": "4ffe1086-25df-4c85-b168-979b5bbf0a26", - "type": "reply-node", - "x": 2170, - "y": 2480, - "properties": { - "config": { - "fields": [ - { - "label": "Content", - "value": "answer" - } - ] - }, - "height": 378, - "stepName": "Specified Reply", - "node_data": { - "fields": [ - "b931efe5-5b66-46e0-ae3b-0160cb18eeb5", - "directly_return" - ], - "content": "", - "reply_type": "referencing", - "is_result": true - } - } - }, - { - "id": "f1f1ee18-5a02-46f6-b4e6-226253cdffbb", - "type": "ai-chat-node", - "x": 2160, - "y": 3200, - "properties": { - "config": { - "fields": [ - { - "label": "AI Answer Content", - "value": "answer" - } - ] - }, - "height": 763, - "stepName": "AI Chat", - "node_data": { - "prompt": "Known information:\n{{Knowledge Search.data}}\nQuestion:\n{{Start.question}}", - "system": "", - "model_id": "", - "dialogue_number": 0, - "is_result": true - } - } - }, - { - "id": "309d0eef-c597-46b5-8d51-b9a28aaef4c7", - "type": "ai-chat-node", - "x": 2160, - "y": 3970, - "properties": { - "config": { - "fields": [ - { - "label": "AI Answer Content", - "value": "answer" - } - ] - }, - "height": 763, - "stepName": "AI Chat1", - "node_data": { - "prompt": "{{Start.question}}", - "system": "", - "model_id": "", - "dialogue_number": 0, - "is_result": true - } - } - } - ], - "edges": [ - { - "id": "7d0f166f-c472-41b2-b9a2-c294f4c83d73", - "type": "app-edge", - "sourceNodeId": "start-node", - "targetNodeId": "b931efe5-5b66-46e0-ae3b-0160cb18eeb5", - "startPoint": { - "x": 590, - "y": 3660 - }, - "endPoint": { - "x": 680, - "y": 3210 - }, - "properties": { - - }, - "pointsList": [ - { - "x": 590, - "y": 3660 - }, - { - "x": 700, - "y": 3660 - }, - { - "x": 570, - "y": 3210 - }, - { - "x": 680, - "y": 3210 - } - ], - "sourceAnchorId": "start-node_right", - "targetAnchorId": "b931efe5-5b66-46e0-ae3b-0160cb18eeb5_left" - }, - { - "id": "35cb86dd-f328-429e-a973-12fd7218b696", - "type": "app-edge", - "sourceNodeId": "b931efe5-5b66-46e0-ae3b-0160cb18eeb5", - "targetNodeId": "fc60863a-dec2-4854-9e5a-7a44b7187a2b", - "startPoint": { - "x": 1000, - "y": 3210 - }, - "endPoint": { - "x": 1200, - "y": 3210 - }, - "properties": { - - }, - "pointsList": [ - { - "x": 1000, - "y": 3210 - }, - { - "x": 1110, - "y": 3210 - }, - { - "x": 1090, - "y": 3210 - }, - { - "x": 1200, - "y": 3210 - } - ], - "sourceAnchorId": "b931efe5-5b66-46e0-ae3b-0160cb18eeb5_right", - "targetAnchorId": "fc60863a-dec2-4854-9e5a-7a44b7187a2b_left" - }, - { - "id": "e8f6cfe6-7e48-41cd-abd3-abfb5304d0d8", - "type": "app-edge", - "sourceNodeId": "fc60863a-dec2-4854-9e5a-7a44b7187a2b", - "targetNodeId": "4ffe1086-25df-4c85-b168-979b5bbf0a26", - "startPoint": { - "x": 1780, - "y": 3073.775 - }, - "endPoint": { - "x": 2010, - "y": 2480 - }, - "properties": { - - }, - "pointsList": [ - { - "x": 1780, - "y": 3073.775 - }, - { - "x": 1890, - "y": 3073.775 - }, - { - "x": 1900, - "y": 2480 - }, - { - "x": 2010, - "y": 2480 - } - ], - "sourceAnchorId": "fc60863a-dec2-4854-9e5a-7a44b7187a2b_1009_right", - "targetAnchorId": "4ffe1086-25df-4c85-b168-979b5bbf0a26_left" - }, - { - "id": "994ff325-6f7a-4ebc-b61b-10e15519d6d2", - "type": "app-edge", - "sourceNodeId": "fc60863a-dec2-4854-9e5a-7a44b7187a2b", - "targetNodeId": "f1f1ee18-5a02-46f6-b4e6-226253cdffbb", - "startPoint": { - "x": 1780, - "y": 3203 - }, - "endPoint": { - "x": 2000, - "y": 3200 - }, - "properties": { - - }, - "pointsList": [ - { - "x": 1780, - "y": 3203 - }, - { - "x": 1890, - "y": 3203 - }, - { - "x": 1890, - "y": 3200 - }, - { - "x": 2000, - "y": 3200 - } - ], - "sourceAnchorId": "fc60863a-dec2-4854-9e5a-7a44b7187a2b_4908_right", - "targetAnchorId": "f1f1ee18-5a02-46f6-b4e6-226253cdffbb_left" - }, - { - "id": "19270caf-bb9f-4ba7-9bf8-200aa70fecd5", - "type": "app-edge", - "sourceNodeId": "fc60863a-dec2-4854-9e5a-7a44b7187a2b", - "targetNodeId": "309d0eef-c597-46b5-8d51-b9a28aaef4c7", - "startPoint": { - "x": 1780, - "y": 3293.6124999999997 - }, - "endPoint": { - "x": 2000, - "y": 3970 - }, - "properties": { - - }, - "pointsList": [ - { - "x": 1780, - "y": 3293.6124999999997 - }, - { - "x": 1890, - "y": 3293.6124999999997 - }, - { - "x": 1890, - "y": 3970 - }, - { - "x": 2000, - "y": 3970 - } - ], - "sourceAnchorId": "fc60863a-dec2-4854-9e5a-7a44b7187a2b_161_right", - "targetAnchorId": "309d0eef-c597-46b5-8d51-b9a28aaef4c7_left" - } - ] -} \ No newline at end of file diff --git a/apps/application/flow/default_workflow_zh.json b/apps/application/flow/default_workflow_zh.json deleted file mode 100644 index 48ac23c4dc6..00000000000 --- a/apps/application/flow/default_workflow_zh.json +++ /dev/null @@ -1,451 +0,0 @@ -{ - "nodes": [ - { - "id": "base-node", - "type": "base-node", - "x": 360, - "y": 2810, - "properties": { - "config": { - - }, - "height": 825.6, - "stepName": "基本信息", - "node_data": { - "desc": "", - "name": "maxkbapplication", - "prologue": "您好，我是 MaxKB 小助手，您可以向我提出 MaxKB 使用问题。\n- MaxKB 主要功能有什么？\n- MaxKB 支持哪些大语言模型？\n- MaxKB 支持哪些文档类型？" - }, - "input_field_list": [ - - ] - } - }, - { - "id": "start-node", - "type": "start-node", - "x": 430, - "y": 3660, - "properties": { - "config": { - "fields": [ - { - "label": "用户问题", - "value": "question" - } - ], - "globalFields": [ - { - "label": "当前时间", - "value": "time" - } - ] - }, - "fields": [ - { - "label": "用户问题", - "value": "question" - } - ], - "height": 276, - "stepName": "开始", - "globalFields": [ - { - "label": "当前时间", - "value": "time" - } - ] - } - }, - { - "id": "b931efe5-5b66-46e0-ae3b-0160cb18eeb5", - "type": "search-dataset-node", - "x": 840, - "y": 3210, - "properties": { - "config": { - "fields": [ - { - "label": "检索结果的分段列表", - "value": "paragraph_list" - }, - { - "label": "满足直接回答的分段列表", - "value": "is_hit_handling_method_list" - }, - { - "label": "检索结果", - "value": "data" - }, - { - "label": "满足直接回答的分段内容", - "value": "directly_return" - } - ] - }, - "height": 794, - "stepName": "知识库检索", - "node_data": { - "dataset_id_list": [ - - ], - "dataset_setting": { - "top_n": 3, - "similarity": 0.6, - "search_mode": "embedding", - "max_paragraph_char_number": 5000 - }, - "question_reference_address": [ - "start-node", - "question" - ], - "source_dataset_id_list": [ - - ] - } - } - }, - { - "id": "fc60863a-dec2-4854-9e5a-7a44b7187a2b", - "type": "condition-node", - "x": 1490, - "y": 3210, - "properties": { - "width": 600, - "config": { - "fields": [ - { - "label": "分支名称", - "value": "branch_name" - } - ] - }, - "height": 543.675, - "stepName": "判断器", - "node_data": { - "branch": [ - { - "id": "1009", - "type": "IF", - "condition": "and", - "conditions": [ - { - "field": [ - "b931efe5-5b66-46e0-ae3b-0160cb18eeb5", - "is_hit_handling_method_list" - ], - "value": "1", - "compare": "len_ge" - } - ] - }, - { - "id": "4908", - "type": "ELSE IF 1", - "condition": "and", - "conditions": [ - { - "field": [ - "b931efe5-5b66-46e0-ae3b-0160cb18eeb5", - "paragraph_list" - ], - "value": "1", - "compare": "len_ge" - } - ] - }, - { - "id": "161", - "type": "ELSE", - "condition": "and", - "conditions": [ - - ] - } - ] - }, - "branch_condition_list": [ - { - "index": 0, - "height": 121.225, - "id": "1009" - }, - { - "index": 1, - "height": 121.225, - "id": "4908" - }, - { - "index": 2, - "height": 44, - "id": "161" - } - ] - } - }, - { - "id": "4ffe1086-25df-4c85-b168-979b5bbf0a26", - "type": "reply-node", - "x": 2170, - "y": 2480, - "properties": { - "config": { - "fields": [ - { - "label": "内容", - "value": "answer" - } - ] - }, - "height": 378, - "stepName": "指定回复", - "node_data": { - "fields": [ - "b931efe5-5b66-46e0-ae3b-0160cb18eeb5", - "directly_return" - ], - "content": "", - "reply_type": "referencing", - "is_result": true - } - } - }, - { - "id": "f1f1ee18-5a02-46f6-b4e6-226253cdffbb", - "type": "ai-chat-node", - "x": 2160, - "y": 3200, - "properties": { - "config": { - "fields": [ - { - "label": "AI 回答内容", - "value": "answer" - } - ] - }, - "height": 763, - "stepName": "AI 对话", - "node_data": { - "prompt": "已知信息：\n{{知识库检索.data}}\n问题：\n{{开始.question}}", - "system": "", - "model_id": "", - "dialogue_number": 0, - "is_result": true - } - } - }, - { - "id": "309d0eef-c597-46b5-8d51-b9a28aaef4c7", - "type": "ai-chat-node", - "x": 2160, - "y": 3970, - "properties": { - "config": { - "fields": [ - { - "label": "AI 回答内容", - "value": "answer" - } - ] - }, - "height": 763, - "stepName": "AI 对话1", - "node_data": { - "prompt": "{{开始.question}}", - "system": "", - "model_id": "", - "dialogue_number": 0, - "is_result": true - } - } - } - ], - "edges": [ - { - "id": "7d0f166f-c472-41b2-b9a2-c294f4c83d73", - "type": "app-edge", - "sourceNodeId": "start-node", - "targetNodeId": "b931efe5-5b66-46e0-ae3b-0160cb18eeb5", - "startPoint": { - "x": 590, - "y": 3660 - }, - "endPoint": { - "x": 680, - "y": 3210 - }, - "properties": { - - }, - "pointsList": [ - { - "x": 590, - "y": 3660 - }, - { - "x": 700, - "y": 3660 - }, - { - "x": 570, - "y": 3210 - }, - { - "x": 680, - "y": 3210 - } - ], - "sourceAnchorId": "start-node_right", - "targetAnchorId": "b931efe5-5b66-46e0-ae3b-0160cb18eeb5_left" - }, - { - "id": "35cb86dd-f328-429e-a973-12fd7218b696", - "type": "app-edge", - "sourceNodeId": "b931efe5-5b66-46e0-ae3b-0160cb18eeb5", - "targetNodeId": "fc60863a-dec2-4854-9e5a-7a44b7187a2b", - "startPoint": { - "x": 1000, - "y": 3210 - }, - "endPoint": { - "x": 1200, - "y": 3210 - }, - "properties": { - - }, - "pointsList": [ - { - "x": 1000, - "y": 3210 - }, - { - "x": 1110, - "y": 3210 - }, - { - "x": 1090, - "y": 3210 - }, - { - "x": 1200, - "y": 3210 - } - ], - "sourceAnchorId": "b931efe5-5b66-46e0-ae3b-0160cb18eeb5_right", - "targetAnchorId": "fc60863a-dec2-4854-9e5a-7a44b7187a2b_left" - }, - { - "id": "e8f6cfe6-7e48-41cd-abd3-abfb5304d0d8", - "type": "app-edge", - "sourceNodeId": "fc60863a-dec2-4854-9e5a-7a44b7187a2b", - "targetNodeId": "4ffe1086-25df-4c85-b168-979b5bbf0a26", - "startPoint": { - "x": 1780, - "y": 3073.775 - }, - "endPoint": { - "x": 2010, - "y": 2480 - }, - "properties": { - - }, - "pointsList": [ - { - "x": 1780, - "y": 3073.775 - }, - { - "x": 1890, - "y": 3073.775 - }, - { - "x": 1900, - "y": 2480 - }, - { - "x": 2010, - "y": 2480 - } - ], - "sourceAnchorId": "fc60863a-dec2-4854-9e5a-7a44b7187a2b_1009_right", - "targetAnchorId": "4ffe1086-25df-4c85-b168-979b5bbf0a26_left" - }, - { - "id": "994ff325-6f7a-4ebc-b61b-10e15519d6d2", - "type": "app-edge", - "sourceNodeId": "fc60863a-dec2-4854-9e5a-7a44b7187a2b", - "targetNodeId": "f1f1ee18-5a02-46f6-b4e6-226253cdffbb", - "startPoint": { - "x": 1780, - "y": 3203 - }, - "endPoint": { - "x": 2000, - "y": 3200 - }, - "properties": { - - }, - "pointsList": [ - { - "x": 1780, - "y": 3203 - }, - { - "x": 1890, - "y": 3203 - }, - { - "x": 1890, - "y": 3200 - }, - { - "x": 2000, - "y": 3200 - } - ], - "sourceAnchorId": "fc60863a-dec2-4854-9e5a-7a44b7187a2b_4908_right", - "targetAnchorId": "f1f1ee18-5a02-46f6-b4e6-226253cdffbb_left" - }, - { - "id": "19270caf-bb9f-4ba7-9bf8-200aa70fecd5", - "type": "app-edge", - "sourceNodeId": "fc60863a-dec2-4854-9e5a-7a44b7187a2b", - "targetNodeId": "309d0eef-c597-46b5-8d51-b9a28aaef4c7", - "startPoint": { - "x": 1780, - "y": 3293.6124999999997 - }, - "endPoint": { - "x": 2000, - "y": 3970 - }, - "properties": { - - }, - "pointsList": [ - { - "x": 1780, - "y": 3293.6124999999997 - }, - { - "x": 1890, - "y": 3293.6124999999997 - }, - { - "x": 1890, - "y": 3970 - }, - { - "x": 2000, - "y": 3970 - } - ], - "sourceAnchorId": "fc60863a-dec2-4854-9e5a-7a44b7187a2b_161_right", - "targetAnchorId": "309d0eef-c597-46b5-8d51-b9a28aaef4c7_left" - } - ] -} \ No newline at end of file diff --git a/apps/application/flow/default_workflow_zh_Hant.json b/apps/application/flow/default_workflow_zh_Hant.json deleted file mode 100644 index 9cac9a54dc6..00000000000 --- a/apps/application/flow/default_workflow_zh_Hant.json +++ /dev/null @@ -1,451 +0,0 @@ -{ - "nodes": [ - { - "id": "base-node", - "type": "base-node", - "x": 360, - "y": 2810, - "properties": { - "config": { - - }, - "height": 825.6, - "stepName": "基本資訊", - "node_data": { - "desc": "", - "name": "maxkbapplication", - "prologue": "您好，我是 MaxKB 小助手，您可以向我提出 MaxKB 使用問題。\n- MaxKB 主要功能有哪些？\n- MaxKB 支援哪些大型語言模型？\n- MaxKB 支援哪些文件類型？" - }, - "input_field_list": [ - - ] - } - }, - { - "id": "start-node", - "type": "start-node", - "x": 430, - "y": 3660, - "properties": { - "config": { - "fields": [ - { - "label": "用戶問題", - "value": "question" - } - ], - "globalFields": [ - { - "label": "當前時間", - "value": "time" - } - ] - }, - "fields": [ - { - "label": "用戶問題", - "value": "question" - } - ], - "height": 276, - "stepName": "開始", - "globalFields": [ - { - "label": "當前時間", - "value": "time" - } - ] - } - }, - { - "id": "b931efe5-5b66-46e0-ae3b-0160cb18eeb5", - "type": "search-dataset-node", - "x": 840, - "y": 3210, - "properties": { - "config": { - "fields": [ - { - "label": "檢索結果的分段列表", - "value": "paragraph_list" - }, - { - "label": "滿足直接回答的分段列表", - "value": "is_hit_handling_method_list" - }, - { - "label": "檢索結果", - "value": "data" - }, - { - "label": "滿足直接回答的分段內容", - "value": "directly_return" - } - ] - }, - "height": 794, - "stepName": "知識庫檢索", - "node_data": { - "dataset_id_list": [ - - ], - "dataset_setting": { - "top_n": 3, - "similarity": 0.6, - "search_mode": "embedding", - "max_paragraph_char_number": 5000 - }, - "question_reference_address": [ - "start-node", - "question" - ], - "source_dataset_id_list": [ - - ] - } - } - }, - { - "id": "fc60863a-dec2-4854-9e5a-7a44b7187a2b", - "type": "condition-node", - "x": 1490, - "y": 3210, - "properties": { - "width": 600, - "config": { - "fields": [ - { - "label": "分支名稱", - "value": "branch_name" - } - ] - }, - "height": 543.675, - "stepName": "判斷器", - "node_data": { - "branch": [ - { - "id": "1009", - "type": "IF", - "condition": "and", - "conditions": [ - { - "field": [ - "b931efe5-5b66-46e0-ae3b-0160cb18eeb5", - "is_hit_handling_method_list" - ], - "value": "1", - "compare": "len_ge" - } - ] - }, - { - "id": "4908", - "type": "ELSE IF 1", - "condition": "and", - "conditions": [ - { - "field": [ - "b931efe5-5b66-46e0-ae3b-0160cb18eeb5", - "paragraph_list" - ], - "value": "1", - "compare": "len_ge" - } - ] - }, - { - "id": "161", - "type": "ELSE", - "condition": "and", - "conditions": [ - - ] - } - ] - }, - "branch_condition_list": [ - { - "index": 0, - "height": 121.225, - "id": "1009" - }, - { - "index": 1, - "height": 121.225, - "id": "4908" - }, - { - "index": 2, - "height": 44, - "id": "161" - } - ] - } - }, - { - "id": "4ffe1086-25df-4c85-b168-979b5bbf0a26", - "type": "reply-node", - "x": 2170, - "y": 2480, - "properties": { - "config": { - "fields": [ - { - "label": "內容", - "value": "answer" - } - ] - }, - "height": 378, - "stepName": "指定回覆", - "node_data": { - "fields": [ - "b931efe5-5b66-46e0-ae3b-0160cb18eeb5", - "directly_return" - ], - "content": "", - "reply_type": "referencing", - "is_result": true - } - } - }, - { - "id": "f1f1ee18-5a02-46f6-b4e6-226253cdffbb", - "type": "ai-chat-node", - "x": 2160, - "y": 3200, - "properties": { - "config": { - "fields": [ - { - "label": "AI 回答內容", - "value": "answer" - } - ] - }, - "height": 763, - "stepName": "AI 對話", - "node_data": { - "prompt": "已知資訊：\n{{知識庫檢索.data}}\n問題：\n{{開始.question}}", - "system": "", - "model_id": "", - "dialogue_number": 0, - "is_result": true - } - } - }, - { - "id": "309d0eef-c597-46b5-8d51-b9a28aaef4c7", - "type": "ai-chat-node", - "x": 2160, - "y": 3970, - "properties": { - "config": { - "fields": [ - { - "label": "AI 回答內容", - "value": "answer" - } - ] - }, - "height": 763, - "stepName": "AI 對話1", - "node_data": { - "prompt": "{{開始.question}}", - "system": "", - "model_id": "", - "dialogue_number": 0, - "is_result": true - } - } - } - ], - "edges": [ - { - "id": "7d0f166f-c472-41b2-b9a2-c294f4c83d73", - "type": "app-edge", - "sourceNodeId": "start-node", - "targetNodeId": "b931efe5-5b66-46e0-ae3b-0160cb18eeb5", - "startPoint": { - "x": 590, - "y": 3660 - }, - "endPoint": { - "x": 680, - "y": 3210 - }, - "properties": { - - }, - "pointsList": [ - { - "x": 590, - "y": 3660 - }, - { - "x": 700, - "y": 3660 - }, - { - "x": 570, - "y": 3210 - }, - { - "x": 680, - "y": 3210 - } - ], - "sourceAnchorId": "start-node_right", - "targetAnchorId": "b931efe5-5b66-46e0-ae3b-0160cb18eeb5_left" - }, - { - "id": "35cb86dd-f328-429e-a973-12fd7218b696", - "type": "app-edge", - "sourceNodeId": "b931efe5-5b66-46e0-ae3b-0160cb18eeb5", - "targetNodeId": "fc60863a-dec2-4854-9e5a-7a44b7187a2b", - "startPoint": { - "x": 1000, - "y": 3210 - }, - "endPoint": { - "x": 1200, - "y": 3210 - }, - "properties": { - - }, - "pointsList": [ - { - "x": 1000, - "y": 3210 - }, - { - "x": 1110, - "y": 3210 - }, - { - "x": 1090, - "y": 3210 - }, - { - "x": 1200, - "y": 3210 - } - ], - "sourceAnchorId": "b931efe5-5b66-46e0-ae3b-0160cb18eeb5_right", - "targetAnchorId": "fc60863a-dec2-4854-9e5a-7a44b7187a2b_left" - }, - { - "id": "e8f6cfe6-7e48-41cd-abd3-abfb5304d0d8", - "type": "app-edge", - "sourceNodeId": "fc60863a-dec2-4854-9e5a-7a44b7187a2b", - "targetNodeId": "4ffe1086-25df-4c85-b168-979b5bbf0a26", - "startPoint": { - "x": 1780, - "y": 3073.775 - }, - "endPoint": { - "x": 2010, - "y": 2480 - }, - "properties": { - - }, - "pointsList": [ - { - "x": 1780, - "y": 3073.775 - }, - { - "x": 1890, - "y": 3073.775 - }, - { - "x": 1900, - "y": 2480 - }, - { - "x": 2010, - "y": 2480 - } - ], - "sourceAnchorId": "fc60863a-dec2-4854-9e5a-7a44b7187a2b_1009_right", - "targetAnchorId": "4ffe1086-25df-4c85-b168-979b5bbf0a26_left" - }, - { - "id": "994ff325-6f7a-4ebc-b61b-10e15519d6d2", - "type": "app-edge", - "sourceNodeId": "fc60863a-dec2-4854-9e5a-7a44b7187a2b", - "targetNodeId": "f1f1ee18-5a02-46f6-b4e6-226253cdffbb", - "startPoint": { - "x": 1780, - "y": 3203 - }, - "endPoint": { - "x": 2000, - "y": 3200 - }, - "properties": { - - }, - "pointsList": [ - { - "x": 1780, - "y": 3203 - }, - { - "x": 1890, - "y": 3203 - }, - { - "x": 1890, - "y": 3200 - }, - { - "x": 2000, - "y": 3200 - } - ], - "sourceAnchorId": "fc60863a-dec2-4854-9e5a-7a44b7187a2b_4908_right", - "targetAnchorId": "f1f1ee18-5a02-46f6-b4e6-226253cdffbb_left" - }, - { - "id": "19270caf-bb9f-4ba7-9bf8-200aa70fecd5", - "type": "app-edge", - "sourceNodeId": "fc60863a-dec2-4854-9e5a-7a44b7187a2b", - "targetNodeId": "309d0eef-c597-46b5-8d51-b9a28aaef4c7", - "startPoint": { - "x": 1780, - "y": 3293.6124999999997 - }, - "endPoint": { - "x": 2000, - "y": 3970 - }, - "properties": { - - }, - "pointsList": [ - { - "x": 1780, - "y": 3293.6124999999997 - }, - { - "x": 1890, - "y": 3293.6124999999997 - }, - { - "x": 1890, - "y": 3970 - }, - { - "x": 2000, - "y": 3970 - } - ], - "sourceAnchorId": "fc60863a-dec2-4854-9e5a-7a44b7187a2b_161_right", - "targetAnchorId": "309d0eef-c597-46b5-8d51-b9a28aaef4c7_left" - } - ] -} \ No newline at end of file diff --git a/apps/application/flow/i_step_node.py b/apps/application/flow/i_step_node.py deleted file mode 100644 index cbe3a630df4..00000000000 --- a/apps/application/flow/i_step_node.py +++ /dev/null @@ -1,470 +0,0 @@ -# coding=utf-8 -""" -@project: maxkb -@Author：虎 -@file： i_step_node.py -@date：2024/6/3 14:57 -@desc: -""" - -import time -import uuid -from abc import abstractmethod -from hashlib import sha1 -from typing import Dict, List, Type - -from application.flow.common import Answer, NodeChunk -from application.long_term_memory import extract_long_term_memory -from application.models import ApplicationChatUserStats, ChatRecord, ChatUserType -from common.field.common import InstanceField -from django.core import cache -from django.db.models import QuerySet -from knowledge.models import ( - Document, - DocumentResourceType, - KnowledgeSyncLog, - KnowledgeSyncStatus, - KnowledgeSyncType, - KnowledgeType, -) -from knowledge.models.knowledge_action import KnowledgeAction, State -from knowledge.services.workflow_sync import merge_workflow_incremental_snapshot -from rest_framework import serializers -from rest_framework.exceptions import ErrorDetail, ValidationError -from tools.models import ToolRecord - -chat_cache = cache - - -def write_context(step_variable: Dict, global_variable: Dict, node, workflow): - if step_variable is not None: - for key in step_variable: - node.context[key] = step_variable[key] - if workflow.is_result(node, NodeResult(step_variable, global_variable)) and "answer" in step_variable: - answer = step_variable["answer"] - yield answer - node.answer_text = answer - if global_variable is not None: - for key in global_variable: - workflow.context[key] = global_variable[key] - node.context["run_time"] = time.time() - node.context["start_time"] - - -def is_interrupt(node, step_variable: Dict, global_variable: Dict): - return node.type == "form-node" and not node.context.get("is_submit", False) - - -class WorkFlowPostHandler: - def __init__(self, chat_info): - self.chat_info = chat_info - - def handler(self, workflow): - workflow_body = workflow.get_body() - question = workflow_body.get("question") - chat_record_id = workflow_body.get("chat_record_id") - chat_id = workflow_body.get("chat_id") - details = workflow.get_runtime_details() - message_tokens = sum( - [ - row.get("message_tokens") - for row in details.values() - if "message_tokens" in row and row.get("message_tokens") is not None - ] - ) - answer_tokens = sum( - [ - row.get("answer_tokens") - for row in details.values() - if "answer_tokens" in row and row.get("answer_tokens") is not None - ] - ) - answer_text_list = workflow.get_answer_text_list() - answer_text = "\n\n".join("\n\n".join([a.get("content") for a in answer]) for answer in answer_text_list) - if workflow.chat_record is not None: - chat_record = workflow.chat_record - chat_record.problem_text = question - chat_record.answer_text = answer_text - chat_record.details = details - chat_record.message_tokens = message_tokens - chat_record.answer_tokens = answer_tokens - chat_record.answer_text_list = answer_text_list - chat_record.run_time = time.time() - workflow.context["start_time"] - else: - chat_record = ChatRecord( - id=chat_record_id, - chat_id=chat_id, - problem_text=question, - answer_text=answer_text, - details=details, - message_tokens=message_tokens, - answer_tokens=answer_tokens, - answer_text_list=answer_text_list, - run_time=time.time() - workflow.context.get("start_time") - if workflow.context.get("start_time") is not None - else 0, - index=0, - ip_address=self.chat_info.ip_address, - source=self.chat_info.source, - ) - - self.chat_info.append_chat_record(chat_record) - self.chat_info.set_cache() - - if not self.chat_info.debug and [ChatUserType.ANONYMOUS_USER.value, ChatUserType.CHAT_USER.value].__contains__( - workflow_body.get("chat_user_type") - ): - application_public_access_client = ( - QuerySet(ApplicationChatUserStats) - .filter( - chat_user_id=workflow_body.get("chat_user_id"), - chat_user_type=workflow_body.get("chat_user_type"), - application_id=self.chat_info.application_id, - ) - .first() - ) - if application_public_access_client is not None: - application_public_access_client.access_num = application_public_access_client.access_num + 1 - application_public_access_client.intraday_access_num = ( - application_public_access_client.intraday_access_num + 1 - ) - application_public_access_client.save() - self.chat_info = None - - extract_long_term_memory.apply_async( - args=( - workflow_body.get("workspace_id"), - workflow_body.get("application_id"), - workflow_body.get("chat_user_id"), - ), - countdown=1, - ) - - -class KnowledgeWorkflowPostHandler(WorkFlowPostHandler): - def __init__(self, chat_info, knowledge_action_id, sync_log_id=None, document_cleanup=None): - super().__init__(chat_info) - self.knowledge_action_id = knowledge_action_id - self.sync_log_id = sync_log_id - self.document_cleanup = document_cleanup - - def handler(self, workflow): - state = get_workflow_state(workflow) - run_time = ( - time.time() - workflow.context.get("start_time") if workflow.context.get("start_time") is not None else 0 - ) - QuerySet(KnowledgeAction).filter(id=self.knowledge_action_id).update(state=state, run_time=run_time) - if self.sync_log_id is not None: - sync_log = QuerySet(KnowledgeSyncLog).filter(id=self.sync_log_id).first() - if sync_log is not None: - if ( - state == State.SUCCESS - and sync_log.sync_type == KnowledgeSyncType.INCREMENTAL - and self.document_cleanup is not None - ): - stats = merge_workflow_incremental_snapshot(sync_log) - else: - stats = { - "total_count": QuerySet(Document) - .filter( - knowledge_id=sync_log.knowledge_id, - resource_type=DocumentResourceType.DOCUMENT, - ) - .count(), - "synced_count": QuerySet(Document) - .filter( - knowledge_id=sync_log.knowledge_id, - type=KnowledgeType.WORKFLOW, - resource_type=DocumentResourceType.DOCUMENT, - create_time__gte=sync_log.create_time, - ) - .count(), - "skipped_count": 0, - "deleted_count": sync_log.deleted_count, - "failed_count": 0 if state == State.SUCCESS else 1, - } - is_success = state == State.SUCCESS - QuerySet(KnowledgeSyncLog).filter(id=sync_log.id).update( - status=KnowledgeSyncStatus.SUCCESS - if is_success and not stats["failed_count"] - else KnowledgeSyncStatus.FAILURE, - total_count=stats["total_count"], - synced_count=stats["synced_count"], - skipped_count=stats["skipped_count"], - deleted_count=stats["deleted_count"], - failed_count=stats["failed_count"], - duration_ms=max(0, round(run_time * 1000)), - message=f"Workflow action {self.knowledge_action_id}: {state}", - ) - - -def get_tool_workflow_state(workflow): - if workflow.is_the_task_interrupted(): - return State.REVOKED - details = workflow.get_runtime_details() - node_list = details.values() - all_node = [*node_list, *get_loop_workflow_node(node_list)] - err = any([True for value in all_node if value.get("status") == 500 and not value.get("enableException")]) - if err: - return State.FAILURE - return State.SUCCESS - - -class ToolWorkflowCallPostHandler(WorkFlowPostHandler): - def __init__(self, chat_info, tool_id): - super().__init__(chat_info) - self.tool_id = tool_id - - def handler(self, workflow): - self.chat_info = None - self.tool_id = None - - -class ToolWorkflowPostHandler(WorkFlowPostHandler): - def __init__(self, chat_info, tool_id): - super().__init__(chat_info) - self.tool_id = tool_id - - def handler(self, workflow): - state = get_tool_workflow_state(workflow) - record = ToolRecord( - id=self.chat_info.tool_record_id, - tool_id=self.tool_id, - workspace_id=self.chat_info.workspace_id, - source_type=self.chat_info.source_type, - source_id=self.chat_info.source_id, - state=state, - run_time=time.time() - workflow.context.get("start_time") - if workflow.context.get("start_time") is not None - else 0, - meta={ - "input_field_list": workflow.get_input_field_list(), - "output_field_list": workflow.get_output_field_list(), - "input": workflow.get_input(), - "output": workflow.out_context, - "details": workflow.get_runtime_details(), - "answer_text_list": workflow.get_answer_text_list(), - }, - ) - self.chat_info.set_record(record) - self.chat_info = None - self.tool_id = None - - -def get_loop_workflow_node(node_list): - result = [] - for item in node_list: - if item.get("type") == "loop-node": - for loop_item in item.get("loop_node_data") or []: - for inner_item in loop_item.values(): - result.append(inner_item) - return result - - -def get_workflow_state(workflow): - if workflow.is_the_task_interrupted(): - return State.REVOKED - details = workflow.get_runtime_details() - node_list = details.values() - all_node = [*node_list, *get_loop_workflow_node(node_list)] - err = any([True for value in all_node if value.get("status") == 500 and not value.get("enableException")]) - if err: - return State.FAILURE - write_is_exist = any([True for value in all_node if value.get("type") == "knowledge-write-node"]) - if not write_is_exist: - return State.FAILURE - return State.SUCCESS - - -class NodeResult: - def __init__( - self, node_variable: Dict, workflow_variable: Dict, _write_context=write_context, _is_interrupt=is_interrupt - ): - self._write_context = _write_context - self.node_variable = node_variable - self.workflow_variable = workflow_variable - self._is_interrupt = _is_interrupt - - def write_context(self, node, workflow): - return self._write_context(self.node_variable, self.workflow_variable, node, workflow) - - def is_assertion_result(self): - return "branch_id" in self.node_variable - - def is_interrupt_exec(self, current_node): - """ - 是否中断执行 - @param current_node: - @return: - """ - return self._is_interrupt(current_node, self.node_variable, self.workflow_variable) - - -class ReferenceAddressSerializer(serializers.Serializer): - node_id = serializers.CharField(required=True, label="节点id") - fields = serializers.ListField( - child=serializers.CharField(required=True, label="节点字段"), required=True, label="节点字段数组" - ) - - -class FlowParamsSerializer(serializers.Serializer): - # 历史对答 - history_chat_record = serializers.ListField( - child=InstanceField(model_type=ChatRecord, required=True), label="历史对答" - ) - - question = serializers.CharField(required=True, label="用户问题") - - chat_id = serializers.CharField(required=True, label="对话id") - - chat_record_id = serializers.CharField(required=True, label="对话记录id") - - stream = serializers.BooleanField(required=True, label="流式输出") - - chat_user_id = serializers.CharField(required=False, label="对话用户id") - - chat_user_type = serializers.CharField(required=False, label="对话用户类型") - - workspace_id = serializers.CharField(required=True, label="工作空间id") - - application_id = serializers.CharField(required=True, label="应用id") - - re_chat = serializers.BooleanField(required=True, label="换个答案") - - debug = serializers.BooleanField(required=True, label="是否debug") - - -class KnowledgeFlowParamsSerializer(serializers.Serializer): - knowledge_id = serializers.UUIDField(required=True, label="知识库id") - workspace_id = serializers.CharField(required=True, label="工作空间id") - knowledge_action_id = serializers.UUIDField(required=True, label="知识库任务执行器id") - data_source = serializers.DictField(required=True, label="数据源") - knowledge_base = serializers.DictField(required=False, label="知识库设置") - user_id = serializers.UUIDField(required=False, label="创建人") - - -class ToolFlowParamsSerializer(serializers.Serializer): - tool_id = serializers.UUIDField(required=True, label="工具id") - workspace_id = serializers.CharField(required=True, label="工作空间id") - - -class INode: - view_type = "many_view" - - @abstractmethod - def save_context(self, details, workflow_manage): - pass - - def get_answer_list(self) -> List[Answer] | None: - if self.answer_text is None: - return None - reasoning_content_enable = self.context.get("model_setting", {}).get("reasoning_content_enable", False) - return [ - Answer( - self.answer_text, - self.view_type, - self.runtime_node_id, - self.workflow_params.get("chat_record_id"), - {}, - self.runtime_node_id, - self.context.get("reasoning_content", "") if reasoning_content_enable else "", - ) - ] - - def __init__( - self, - node, - workflow_params, - workflow_manage, - up_node_id_list=None, - get_node_params=lambda node: node.properties.get("node_data"), - salt=None, - ): - # 当前步骤上下文,用于存储当前步骤信息 - self.status = 200 - self.err_message = "" - self.node = node - self.node_params = get_node_params(node) - self.workflow_params = workflow_params - self.workflow_manage = workflow_manage - self.node_params_serializer = None - self.flow_params_serializer = None - self.context = {} - self.answer_text = None - self.id = node.id - if up_node_id_list is None: - up_node_id_list = [] - self.up_node_id_list = up_node_id_list - self.node_chunk = NodeChunk() - self.runtime_node_id = sha1( - uuid.NAMESPACE_DNS.bytes - + bytes(str(uuid.uuid5(uuid.NAMESPACE_DNS, "".join([*sorted(up_node_id_list), node.id]))), "utf-8") - ).hexdigest() + ("__" + str(salt) if salt is not None else "") - self.extra = {} - - def valid_args(self, node_params, flow_params): - flow_params_serializer_class = self.get_flow_params_serializer_class() - node_params_serializer_class = self.get_node_params_serializer_class() - if flow_params_serializer_class is not None and flow_params is not None: - self.flow_params_serializer = flow_params_serializer_class(data=flow_params) - self.flow_params_serializer.is_valid(raise_exception=True) - if node_params_serializer_class is not None: - self.node_params_serializer = node_params_serializer_class(data=node_params) - self.node_params_serializer.is_valid(raise_exception=True) - if self.node.properties.get("status", 200) != 200: - raise ValidationError(ErrorDetail(f"节点{self.node.properties.get('stepName')} 不可用")) - - def get_reference_field(self, fields: List[str]): - return self.get_field(self.context, fields) - - @staticmethod - def get_field(obj, fields: List[str]): - for field in fields: - value = obj.get(field) - if value is None: - return None - else: - obj = value - return obj - - @abstractmethod - def get_node_params_serializer_class(self) -> Type[serializers.Serializer]: - pass - - def get_flow_params_serializer_class(self) -> Type[serializers.Serializer]: - return self.workflow_manage.get_params_serializer_class() - - def get_write_error_context(self, e): - self.status = 500 - self.answer_text = str(e) - self.err_message = str(e) - current_time = time.time() - self.context["run_time"] = current_time - (self.context.get("start_time") or current_time) - - def write_error_context(answer, status=200): - pass - - return write_error_context - - def run(self) -> NodeResult: - """ - :return: 执行结果 - """ - start_time = time.time() - self.context["start_time"] = start_time - result = self._run() - self.context["run_time"] = time.time() - start_time - return result - - def _run(self): - result = self.execute() - return result - - def execute(self, **kwargs) -> NodeResult: - pass - - def get_details(self, index: int, **kwargs): - """ - 运行详情 - :return: 步骤详情 - """ - return {} diff --git a/apps/application/flow/knowledge_loop_workflow_manage.py b/apps/application/flow/knowledge_loop_workflow_manage.py deleted file mode 100644 index 31d3ab4df25..00000000000 --- a/apps/application/flow/knowledge_loop_workflow_manage.py +++ /dev/null @@ -1,21 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： workflow_manage.py - @date：2024/1/9 17:40 - @desc: -""" -from application.flow.i_step_node import KnowledgeFlowParamsSerializer -from application.flow.loop_workflow_manage import LoopWorkflowManage - - -class KnowledgeLoopWorkflowManage(LoopWorkflowManage): - def get_params_serializer_class(self): - return KnowledgeFlowParamsSerializer - - def get_source_type(self): - return "KNOWLEDGE" - - def get_source_id(self): - return self.params.get('knowledge_id') diff --git a/apps/application/flow/knowledge_workflow_manage.py b/apps/application/flow/knowledge_workflow_manage.py deleted file mode 100644 index 98212c9ee5a..00000000000 --- a/apps/application/flow/knowledge_workflow_manage.py +++ /dev/null @@ -1,130 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎虎 - @file： Knowledge_workflow_manage.py - @date：2025/11/13 19:02 - @desc: -""" -import time -import traceback -from concurrent.futures import ThreadPoolExecutor - -from django.db.models import QuerySet -from django.utils.translation import get_language - -from application.flow.common import Workflow -from application.flow.i_step_node import WorkFlowPostHandler, KnowledgeFlowParamsSerializer, NodeResult -from application.flow.workflow_manage import WorkflowManage -from common.handle.base_to_response import BaseToResponse -from common.handle.impl.response.system_to_response import SystemToResponse -from knowledge.models.knowledge_action import KnowledgeAction, State - -executor = ThreadPoolExecutor(max_workers=200) - - -class KnowledgeWorkflowManage(WorkflowManage): - - def __init__(self, flow: Workflow, - params, - work_flow_post_handler: WorkFlowPostHandler, - base_to_response: BaseToResponse = SystemToResponse(), - start_node_id=None, - start_node_data=None, chat_record=None, child_node=None, is_the_task_interrupted=lambda: False): - super().__init__(flow, params, work_flow_post_handler, base_to_response, None, None, None, - None, - None, None, start_node_id, start_node_data, chat_record, child_node, is_the_task_interrupted) - - def get_params_serializer_class(self): - return KnowledgeFlowParamsSerializer - - def get_start_node(self): - start_node_list = [node for node in self.flow.nodes if - self.params.get('data_source', {}).get('node_id') == node.id] - return start_node_list[0] - - def run(self): - self.context['start_time'] = time.time() - executor.submit(self._run) - - def _run(self): - QuerySet(KnowledgeAction).filter(id=self.params.get('knowledge_action_id')).update( - state=State.STARTED) - language = get_language() - self.run_chain_async(self.start_node, None, language) - while self.is_run(): - pass - self.work_flow_post_handler.handler(self) - - @staticmethod - def get_node_details(current_node, node, index): - if current_node == node: - return { - 'name': node.node.properties.get('stepName'), - "index": index, - 'run_time': 0, - 'type': node.type, - 'status': 202, - 'err_message': "" - } - - return node.get_details(index) - - def run_chain(self, current_node, node_result_future=None): - QuerySet(KnowledgeAction).filter(id=self.params.get('knowledge_action_id')).update( - details=self.get_runtime_details(lambda node, index: self.get_node_details(current_node, node, index))) - if node_result_future is None: - node_result_future = self.run_node_future(current_node) - try: - result = self.hand_node_result(current_node, node_result_future) - return result - except Exception as e: - traceback.print_exc() - return None - - def hand_node_result(self, current_node, node_result_future): - try: - current_result = node_result_future.result() - result = current_result.write_context(current_node, self) - if result is not None: - # 阻塞获取结果 - list(result) - if current_node.status == 500: - enableException = current_node.node.properties.get('enableException') - if not enableException: - return None - current_node.context['exception_message'] = current_node.err_message - current_node.context['branch_id'] = 'exception' - r = NodeResult({'branch_id': 'exception', 'exception': current_node.err_message}, {}, - _is_interrupt=lambda node, step_variable, global_variable: False) - r.write_context(current_node, self) - return r - if self.is_the_task_interrupted(): - current_node.status = 201 - return None - return current_result - except Exception as e: - traceback.print_exc() - self.status = 500 - current_node.get_write_error_context(e) - self.answer += str(e) - if self.is_the_task_interrupted(): - current_node.status = 201 - return None - enableException = current_node.node.properties.get('enableException') - if enableException: - current_node.context['exception_message'] = current_node.err_message - current_node.context['branch_id'] = 'exception' - return NodeResult({'branch_id': 'exception', 'exception': current_node.err_message}, {}, - _is_interrupt=lambda node, step_variable, global_variable: False) - QuerySet(KnowledgeAction).filter(id=self.params.get('knowledge_action_id')).update(state=State.FAILURE) - finally: - current_node.node_chunk.end() - QuerySet(KnowledgeAction).filter(id=self.params.get('knowledge_action_id')).update( - details=self.get_runtime_details()) - - def get_source_type(self): - return "KNOWLEDGE" - - def get_source_id(self): - return self.params.get('knowledge_id') diff --git a/apps/application/flow/loop_workflow_manage.py b/apps/application/flow/loop_workflow_manage.py deleted file mode 100644 index c236b15dcc5..00000000000 --- a/apps/application/flow/loop_workflow_manage.py +++ /dev/null @@ -1,199 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： workflow_manage.py - @date：2024/1/9 17:40 - @desc: -""" -from concurrent.futures import ThreadPoolExecutor -from typing import List - -from django.db import close_old_connections -from django.utils.translation import get_language -from langchain_core.prompts import PromptTemplate - -from application.flow.common import Workflow -from application.flow.i_step_node import WorkFlowPostHandler, INode -from application.flow.step_node import get_node -from application.flow.workflow_manage import WorkflowManage -from common.handle.base_to_response import BaseToResponse -from common.handle.impl.response.system_to_response import SystemToResponse - -executor = ThreadPoolExecutor(max_workers=200) - - -class NodeResultFuture: - def __init__(self, r, e, status=200): - self.r = r - self.e = e - self.status = status - - def result(self): - if self.status == 200: - return self.r - else: - raise self.e - - -def await_result(result, timeout=1): - try: - result.result(timeout) - return False - except Exception as e: - return True - - -class NodeChunkManage: - - def __init__(self, work_flow): - self.node_chunk_list = [] - self.current_node_chunk = None - self.work_flow = work_flow - - def add_node_chunk(self, node_chunk): - self.node_chunk_list.append(node_chunk) - - def contains(self, node_chunk): - return self.node_chunk_list.__contains__(node_chunk) - - def pop(self): - if self.current_node_chunk is None: - try: - current_node_chunk = self.node_chunk_list.pop(0) - self.current_node_chunk = current_node_chunk - except IndexError as e: - pass - if self.current_node_chunk is not None: - try: - chunk = self.current_node_chunk.chunk_list.pop(0) - return chunk - except IndexError as e: - if self.current_node_chunk.is_end(): - self.current_node_chunk = None - if self.work_flow.answer_is_not_empty(): - chunk = self.work_flow.base_to_response.to_stream_chunk_response( - self.work_flow.params['chat_id'], - self.work_flow.params['chat_record_id'], - '\n\n', False, 0, 0) - self.work_flow.append_answer('\n\n') - return chunk - return self.pop() - return None - - -class LoopWorkflowManage(WorkflowManage): - - def __init__(self, flow: Workflow, - params, - work_flow_post_handler: WorkFlowPostHandler, - parentWorkflowManage, - loop_params, - get_loop_context, - base_to_response: BaseToResponse = SystemToResponse(), - start_node_id=None, - start_node_data=None, chat_record=None, child_node=None, is_the_task_interrupted=lambda: False): - self.parentWorkflowManage = parentWorkflowManage - self.loop_params = loop_params - self.get_loop_context = get_loop_context - self.loop_field_list = [] - super().__init__(flow, params, work_flow_post_handler, base_to_response, None, None, None, - None, - None, None, start_node_id, start_node_data, chat_record, child_node, is_the_task_interrupted) - - def get_node_cls_by_id(self, node_id, up_node_id_list=None, - get_node_params=lambda node: node.properties.get('node_data')): - for node in self.flow.nodes: - if node.id == node_id: - node_instance = get_node(node.type, self.flow.workflow_mode)(node, - self.params, self, up_node_id_list, - get_node_params, - salt=self.get_index()) - return node_instance - return None - - def stream(self): - close_old_connections() - language = get_language() - self.run_chain_async(self.start_node, None, language) - return self.await_result(is_cleanup=False) - - def get_index(self): - return self.loop_params.get('index') - - def get_start_node(self): - start_node_list = [node for node in self.flow.nodes if - ['loop-start-node'].__contains__(node.type)] - return start_node_list[0] - - def get_reference_field(self, node_id: str, fields: List[str]): - """ - @param node_id: 节点id - @param fields: 字段 - @return: - """ - if node_id == 'global': - return self.parentWorkflowManage.get_reference_field(node_id, fields) - elif node_id == 'chat': - return self.parentWorkflowManage.get_reference_field(node_id, fields) - elif node_id == 'loop': - loop_context = self.get_loop_context() - return INode.get_field(loop_context, fields) - else: - node = self.get_node_by_id(node_id) - if node: - return node.get_reference_field(fields) - return self.parentWorkflowManage.get_reference_field(node_id, fields) - - def get_workflow_content(self): - context = { - 'global': self.context, - 'chat': self.chat_context, - 'loop': self.get_loop_context(), - } - - for node in self.node_context: - context[node.id] = node.context - return context - - def init_fields(self): - super().init_fields() - loop_field_list = [] - loop_start_node = self.flow.get_node('loop-start-node') - loop_input_field_list = loop_start_node.properties.get('loop_input_field_list') - node_name = loop_start_node.properties.get('stepName') - node_id = loop_start_node.id - if loop_input_field_list is not None: - for f in loop_input_field_list: - loop_field_list.append( - {'label': f.get('label'), 'value': f.get('field'), 'node_id': node_id, 'node_name': node_name}) - self.loop_field_list = loop_field_list - - def reset_prompt(self, prompt: str): - prompt = super().reset_prompt(prompt) - for field in self.loop_field_list: - chatLabel = f"loop.{field.get('value')}" - chatValue = f"context.get('loop').get('{field.get('value', '')}','')" - prompt = prompt.replace(chatLabel, chatValue) - - prompt = self.parentWorkflowManage.reset_prompt(prompt) - return prompt - - def generate_prompt(self, prompt: str): - """ - 格式化生成提示词 - @param prompt: 提示词信息 - @return: 格式化后的提示词 - """ - - context = {**self.get_workflow_content(), **self.parentWorkflowManage.get_workflow_content()} - prompt = self.reset_prompt(prompt) - prompt_template = PromptTemplate.from_template(prompt, template_format='jinja2') - value = prompt_template.format(context=context) - return value - - def get_source_type(self): - return "APPLICATION" - - def get_source_id(self): - return self.params.get('application_id') diff --git a/apps/application/flow/step_node/__init__.py b/apps/application/flow/step_node/__init__.py deleted file mode 100644 index 4c38020771e..00000000000 --- a/apps/application/flow/step_node/__init__.py +++ /dev/null @@ -1,63 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： __init__.py.py - @date：2024/6/7 14:43 - @desc: -""" -from .ai_chat_step_node import * -from .application_node import BaseApplicationNode -from .condition_node import * -from .data_source_local_node.impl.base_data_source_local_node import BaseDataSourceLocalNode -from .data_source_web_node.impl.base_data_source_web_node import BaseDataSourceWebNode -from .direct_reply_node import * -from .document_extract_node import * -from .form_node import * -from .image_generate_step_node import * -from .image_to_video_step_node import BaseImageToVideoNode -from .image_understand_step_node import * -from .intent_node import * -from .knowledge_write_node.impl.base_knowledge_write_node import BaseKnowledgeWriteNode -from .loop_break_node import BaseLoopBreakNode -from .loop_continue_node import BaseLoopContinueNode -from .loop_node import * -from .loop_start_node import * -from .mcp_node import BaseMcpNode -from .parameter_extraction_node import BaseParameterExtractionNode -from .question_node import * -from .reranker_node import * -from .search_document_node import BaseSearchDocumentNode -from .search_knowledge_node import * -from .speech_to_text_step_node import BaseSpeechToTextNode -from .start_node import * -from .text_to_speech_step_node.impl.base_text_to_speech_node import BaseTextToSpeechNode -from .text_to_video_step_node.impl.base_text_to_video_node import BaseTextToVideoNode -from .tool_lib_node import * -from .tool_node import * -from .tool_workflow_lib_node import BaseToolWorkflowLibNodeNode -from .variable_aggregation_node.impl.base_variable_aggregation_node import BaseVariableAggregationNode -from .variable_assign_node import BaseVariableAssignNode -from .variable_splitting_node import BaseVariableSplittingNode -from .video_understand_step_node import BaseVideoUnderstandNode -from .document_split_node import BaseDocumentSplitNode -from .tool_start_node import BaseToolStartStepNode - -node_list = [BaseStartStepNode, BaseChatNode, BaseSearchKnowledgeNode, BaseSearchDocumentNode, BaseQuestionNode, - BaseConditionNode, BaseReplyNode, - BaseToolNodeNode, BaseToolLibNodeNode, BaseRerankerNode, BaseApplicationNode, - BaseDocumentExtractNode, - BaseImageUnderstandNode, BaseFormNode, BaseSpeechToTextNode, BaseTextToSpeechNode, - BaseImageGenerateNode, BaseVariableAssignNode, BaseMcpNode, BaseTextToVideoNode, BaseImageToVideoNode, - BaseVideoUnderstandNode, - BaseIntentNode, BaseLoopNode, BaseLoopStartStepNode, - BaseLoopContinueNode, - BaseLoopBreakNode, BaseVariableSplittingNode, BaseParameterExtractionNode, BaseVariableAggregationNode, - BaseDataSourceLocalNode, BaseDataSourceWebNode, BaseKnowledgeWriteNode, BaseDocumentSplitNode, - BaseToolStartStepNode, BaseToolWorkflowLibNodeNode] - -node_map = {n.type: {w: n for w in n.support} for n in node_list} - - -def get_node(node_type, workflow_model): - return node_map.get(node_type).get(workflow_model) diff --git a/apps/application/flow/step_node/ai_chat_step_node/__init__.py b/apps/application/flow/step_node/ai_chat_step_node/__init__.py deleted file mode 100644 index 1929ae2af49..00000000000 --- a/apps/application/flow/step_node/ai_chat_step_node/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： __init__.py - @date：2024/6/11 15:29 - @desc: -""" -from .impl import * diff --git a/apps/application/flow/step_node/ai_chat_step_node/i_chat_node.py b/apps/application/flow/step_node/ai_chat_step_node/i_chat_node.py deleted file mode 100644 index 0483c9cb5e7..00000000000 --- a/apps/application/flow/step_node/ai_chat_step_node/i_chat_node.py +++ /dev/null @@ -1,92 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： i_chat_node.py - @date：2024/6/4 13:58 - @desc: -""" -from typing import Type - -from django.utils.translation import gettext_lazy as _ -from rest_framework import serializers - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import INode, NodeResult - - -class ChatNodeSerializer(serializers.Serializer): - model_id = serializers.CharField(required=False, allow_blank=True, allow_null=True, label=_("Model id")) - model_id_type = serializers.CharField(required=False, default='custom', label=_("Model id type")) - model_id_reference = serializers.ListField(required=False, child=serializers.CharField(), allow_empty=True, - label=_("Reference Field")) - system = serializers.CharField(required=False, allow_blank=True, allow_null=True, - label=_("Role Setting")) - prompt = serializers.CharField(required=True, label=_("Prompt word")) - # 多轮对话数量 - dialogue_number = serializers.IntegerField(required=True, label=_("Number of multi-round conversations")) - - is_result = serializers.BooleanField(required=False, - label=_('Whether to return content')) - - model_params_setting = serializers.DictField(required=False, - label=_("Model parameter settings")) - model_setting = serializers.DictField(required=False, - label='Model settings') - dialogue_type = serializers.CharField(required=False, allow_blank=True, allow_null=True, - label=_("Context Type")) - mcp_servers = serializers.JSONField(required=False, label=_("MCP Server")) - mcp_tool_id = serializers.CharField(required=False, allow_blank=True, allow_null=True, label=_("MCP Tool ID")) - mcp_tool_ids = serializers.ListField(child=serializers.UUIDField(), required=False, allow_empty=True, - label=_("MCP Tool IDs"), ) - mcp_source = serializers.CharField(required=False, allow_blank=True, allow_null=True, label=_("MCP Source")) - - tool_ids = serializers.ListField(child=serializers.UUIDField(), required=False, allow_empty=True, - label=_("Tool IDs"), ) - application_ids = serializers.ListField(child=serializers.UUIDField(), required=False, allow_empty=True, - label=_("App IDs"), ) - skill_tool_ids = serializers.ListField(child=serializers.UUIDField(), required=False, allow_empty=True, - label=_("Skill IDs"), ) - mcp_output_enable = serializers.BooleanField(required=False, default=True, label=_("Whether to enable MCP output")) - - video_list = serializers.ListField(required=False, label=_("video")) - - image_list = serializers.ListField(required=False, label=_("picture")) - - vision = serializers.BooleanField(required=False, default=False, label=_("vision")) - - -class IChatNode(INode): - type = 'ai-chat-node' - support = [WorkflowMode.APPLICATION, WorkflowMode.APPLICATION_LOOP, WorkflowMode.KNOWLEDGE_LOOP, - WorkflowMode.KNOWLEDGE, WorkflowMode.TOOL, WorkflowMode.TOOL_LOOP] - - def get_node_params_serializer_class(self) -> Type[serializers.Serializer]: - return ChatNodeSerializer - - def _run(self): - if [WorkflowMode.KNOWLEDGE, WorkflowMode.KNOWLEDGE_LOOP, WorkflowMode.TOOL, - WorkflowMode.TOOL_LOOP].__contains__( - self.workflow_manage.flow.workflow_mode): - return self.execute(**self.node_params_serializer.data, **self.flow_params_serializer.data, - **{'history_chat_record': [], 'stream': True, 'chat_id': None, 'chat_record_id': None}) - else: - return self.execute(**self.node_params_serializer.data, **self.flow_params_serializer.data) - - def execute(self, model_id, system, prompt, dialogue_number, history_chat_record, stream, chat_id, - chat_record_id, - model_params_setting=None, - model_id_type=None, - model_id_reference=None, - dialogue_type=None, - model_setting=None, - mcp_servers=None, - mcp_tool_id=None, - mcp_tool_ids=None, - mcp_source=None, - tool_ids=None, - application_ids=None, - skill_tool_ids=None, - mcp_output_enable=True, - **kwargs) -> NodeResult: - pass diff --git a/apps/application/flow/step_node/ai_chat_step_node/impl/__init__.py b/apps/application/flow/step_node/ai_chat_step_node/impl/__init__.py deleted file mode 100644 index 79051a999fb..00000000000 --- a/apps/application/flow/step_node/ai_chat_step_node/impl/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： __init__.py - @date：2024/6/11 15:34 - @desc: -""" -from .base_chat_node import BaseChatNode diff --git a/apps/application/flow/step_node/ai_chat_step_node/impl/base_chat_node.py b/apps/application/flow/step_node/ai_chat_step_node/impl/base_chat_node.py deleted file mode 100644 index 860f37eea48..00000000000 --- a/apps/application/flow/step_node/ai_chat_step_node/impl/base_chat_node.py +++ /dev/null @@ -1,624 +0,0 @@ -# coding=utf-8 -""" -@project: maxkb -@Author：虎 -@file： base_question_node.py -@date：2024/6/4 14:30 -@desc: -""" - -import base64 -import json -import re -import time -from functools import reduce -from typing import Dict, List - -from common.exception.app_exception import AppApiException -from common.utils.common import guess_image_format -from common.utils.rsa_util import rsa_long_decrypt -from common.utils.shared_resource_auth import filter_authorized_ids -from common.utils.tool_code import ToolExecutor -from django.db.models import QuerySet -from django.utils.translation import gettext as _ -from knowledge.models import File -from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, SystemMessage -from models_provider.models import Model -from models_provider.tools import get_model_credential, get_model_instance_by_model_workspace_id -from tools.models import Tool, ToolType - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import INode, NodeResult -from application.flow.step_node.ai_chat_step_node.i_chat_node import IChatNode -from application.flow.tools import Reasoning, get_tools, mcp_response_generator -from application.models import Application, ApplicationAccessToken, ApplicationApiKey - - -def _write_context( - node_variable: Dict, workflow_variable: Dict, node: INode, workflow, answer: str, reasoning_content: str -): - chat_model = node_variable.get("chat_model") - message_tokens = chat_model.get_num_tokens_from_messages(node_variable.get("message_list")) - answer_tokens = chat_model.get_num_tokens(answer) - node.context["message_tokens"] = message_tokens - node.context["answer_tokens"] = answer_tokens - node.context["answer"] = answer - node.context["question"] = node_variable["question"] - node.context["run_time"] = time.time() - node.context["start_time"] - node.context["reasoning_content"] = reasoning_content - if workflow.is_result(node, NodeResult(node_variable, workflow_variable)): - node.answer_text = answer - - -def write_context_stream(node_variable: Dict, workflow_variable: Dict, node: INode, workflow): - """ - 写入上下文数据 (流式) - @param node_variable: 节点数据 - @param workflow_variable: 全局数据 - @param node: 节点 - @param workflow: 工作流管理器 - """ - response = node_variable.get("result") - answer = "" - reasoning_content = "" - model_setting = node.context.get( - "model_setting", - {"reasoning_content_enable": False, "reasoning_content_end": "", "reasoning_content_start": ""}, - ) - reasoning = Reasoning( - model_setting.get("reasoning_content_start", ""), model_setting.get("reasoning_content_end", "") - ) - response_reasoning_content = False - - for chunk in response: - if workflow.is_the_task_interrupted(): - break - reasoning_chunk = reasoning.get_reasoning_content(chunk) - content_chunk = reasoning_chunk.get("content") - if "reasoning_content" in chunk.additional_kwargs: - response_reasoning_content = True - reasoning_content_chunk = chunk.additional_kwargs.get("reasoning_content", "") - else: - reasoning_content_chunk = reasoning_chunk.get("reasoning_content") - answer += content_chunk - if reasoning_content_chunk is None: - reasoning_content_chunk = "" - reasoning_content += reasoning_content_chunk - yield { - "content": content_chunk, - "reasoning_content": reasoning_content_chunk - if model_setting.get("reasoning_content_enable", False) - else "", - } - - reasoning_chunk = reasoning.get_end_reasoning_content() - answer += reasoning_chunk.get("content") - reasoning_content_chunk = "" - if not response_reasoning_content: - reasoning_content_chunk = reasoning_chunk.get("reasoning_content") - yield { - "content": reasoning_chunk.get("content"), - "reasoning_content": reasoning_content_chunk if model_setting.get("reasoning_content_enable", False) else "", - } - _write_context(node_variable, workflow_variable, node, workflow, answer, reasoning_content) - - -def write_context(node_variable: Dict, workflow_variable: Dict, node: INode, workflow): - """ - 写入上下文数据 - @param node_variable: 节点数据 - @param workflow_variable: 全局数据 - @param node: 节点实例对象 - @param workflow: 工作流管理器 - """ - response = node_variable.get("result") - model_setting = node.context.get( - "model_setting", - {"reasoning_content_enable": False, "reasoning_content_end": "", "reasoning_content_start": ""}, - ) - reasoning = Reasoning(model_setting.get("reasoning_content_start"), model_setting.get("reasoning_content_end")) - reasoning_result = reasoning.get_reasoning_content(response) - reasoning_result_end = reasoning.get_end_reasoning_content() - content = reasoning_result.get("content") + reasoning_result_end.get("content") - meta = {**response.response_metadata, **response.additional_kwargs} - if "reasoning_content" in meta: - reasoning_content = meta.get("reasoning_content", "") or "" - else: - reasoning_content = (reasoning_result.get("reasoning_content") or "") + ( - reasoning_result_end.get("reasoning_content") or "" - ) - _write_context(node_variable, workflow_variable, node, workflow, content, reasoning_content) - - -CHAT_FILE_LIST_FIELDS = ("image_list", "document_list", "audio_list", "video_list", "other_list") - - -def get_default_model_params_setting(model_id): - model = QuerySet(Model).filter(id=model_id).first() - credential = get_model_credential(model.provider, model.model_type, model.model_name) - model_params_setting = credential.get_model_params_setting_form(model.model_name).get_default_form_data() - return model_params_setting - - -def get_node_message(chat_record, runtime_node_id): - node_details = chat_record.get_node_details_runtime_node_id(runtime_node_id) - if node_details is None: - return [] - return [HumanMessage(node_details.get("question")), AIMessage(node_details.get("answer"))] - - -def get_workflow_message(chat_record): - return [chat_record.get_human_message(), chat_record.get_ai_message()] - - -def get_message(chat_record, dialogue_type, runtime_node_id): - return ( - get_node_message(chat_record, runtime_node_id) if dialogue_type == "NODE" else get_workflow_message(chat_record) - ) - - -class BaseChatNode(IChatNode): - def save_context(self, details, workflow_manage): - self.context["answer"] = details.get("answer") - self.context["question"] = details.get("question") - self.context["reasoning_content"] = details.get("reasoning_content") - self.context["exception_message"] = details.get("err_message") - if self.node_params.get("is_result", False): - self.answer_text = details.get("answer") - - def execute( - self, - model_id, - system, - prompt, - dialogue_number, - history_chat_record, - stream, - chat_id, - chat_record_id, - model_params_setting=None, - model_id_type=None, - model_id_reference=None, - dialogue_type=None, - model_setting=None, - mcp_servers=None, - mcp_tool_id=None, - mcp_tool_ids=None, - mcp_source=None, - tool_ids=None, - application_ids=None, - skill_tool_ids=None, - mcp_output_enable=True, - **kwargs, - ) -> NodeResult: - if dialogue_type is None: - dialogue_type = "WORKFLOW" - - if model_id_type == "reference" and model_id_reference: - reference_data = self.workflow_manage.get_reference_field( - model_id_reference[0], - model_id_reference[1:], - ) - - if reference_data and isinstance(reference_data, dict): - model_id = reference_data.get("model_id", model_id) - model_params_setting = reference_data.get("model_params_setting") - if model_id is None or model_id == "": - raise Exception(_("Model is not allowed to be empty")) - - if model_params_setting is None and model_id: - model_params_setting = get_default_model_params_setting(model_id) - - if model_setting is None: - model_setting = { - "reasoning_content_enable": False, - "reasoning_content_end": "", - "reasoning_content_start": "", - } - self.context["model_setting"] = model_setting - workspace_id = self.workflow_manage.get_body().get("workspace_id") - chat_model = get_model_instance_by_model_workspace_id(model_id, workspace_id, **(model_params_setting or {})) - history_message = self.get_history_message( - history_chat_record, dialogue_number, dialogue_type, self.runtime_node_id - ) - self.context["history_message"] = [ - {"content": message.content, "role": message.type} - for message in (history_message if history_message is not None else []) - ] - question = self.generate_prompt_question(prompt, chat_model) - self.context["question"] = question.content - system = self.workflow_manage.generate_prompt(system) - self.context["system"] = system - message_list = self.generate_message_list(question, history_message) - self.context["message_list"] = message_list - - # 过滤tool_id - all_tool_ids = list( - set( - (mcp_tool_ids or []) - + (tool_ids or []) - + (skill_tool_ids or []) - + ([mcp_tool_id] if mcp_tool_id else []) - ) - ) - authorized_set = set(filter_authorized_ids("tool", all_tool_ids, workspace_id)) - - mcp_tool_ids = [i for i in (mcp_tool_ids or []) if i in authorized_set] - tool_ids = [i for i in (tool_ids or []) if i in authorized_set] - skill_tool_ids = [i for i in (skill_tool_ids or []) if i in authorized_set] - mcp_tool_id = mcp_tool_id if (mcp_tool_id and mcp_tool_id in authorized_set) else None - # 处理 MCP 请求 - mcp_result = self._handle_mcp_request( - mcp_source, - mcp_servers, - mcp_tool_id, - mcp_tool_ids, - tool_ids, - application_ids, - skill_tool_ids, - mcp_output_enable, - chat_model, - SystemMessage(system), - message_list, - history_message, - question, - chat_id, - workspace_id, - ) - if mcp_result: - return mcp_result - message_list = [SystemMessage(system)] + message_list - if stream: - r = chat_model.stream(message_list) - return NodeResult( - {"result": r, "chat_model": chat_model, "message_list": message_list, "question": question.content}, - {}, - _write_context=write_context_stream, - ) - else: - r = chat_model.invoke(message_list) - return NodeResult( - { - "result": r, - "chat_model": chat_model, - "message_list": message_list, - "history_message": [ - {"content": message.content, "role": message.type} - for message in (history_message if history_message is not None else []) - ], - "question": question.content, - }, - {}, - _write_context=write_context, - ) - - def _handle_mcp_request( - self, - mcp_source, - mcp_servers, - mcp_tool_id, - mcp_tool_ids, - tool_ids, - application_ids, - skill_tool_ids, - mcp_output_enable, - chat_model, - system_prompt, - message_list, - history_message, - question, - chat_id, - workspace_id, - ): - - mcp_servers_config = {} - - # 迁移过来mcp_source是None - if mcp_source is None: - mcp_source = "custom" - # 兼容老数据 - if not mcp_tool_ids: - mcp_tool_ids = [] - if mcp_tool_id: - mcp_tool_ids = list(set(mcp_tool_ids + [mcp_tool_id])) - if mcp_source == "custom" and mcp_servers: - mcp_servers_config = json.loads(mcp_servers) - mcp_servers_config = self.handle_variables(mcp_servers_config) - elif mcp_tool_ids: - mcp_tools = QuerySet(Tool).filter(id__in=mcp_tool_ids).values() - for mcp_tool in mcp_tools: - if mcp_tool and mcp_tool["is_active"]: - mcp_servers_config = {**mcp_servers_config, **json.loads(mcp_tool["code"])} - mcp_servers_config = self.handle_variables(mcp_servers_config) - # 校验代码是否包括禁止的关键字 - ToolExecutor().validate_mcp_transport(json.dumps(mcp_servers_config)) - - tool_init_params = {} - tools = get_tools( - self.workflow_manage.get_source_type(), self.workflow_manage.get_source_id(), tool_ids, workspace_id - ) - if tool_ids and len(tool_ids) > 0: # 如果有工具ID，则将其转换为MCP - self.context["tool_ids"] = tool_ids - custom_tools_map = { - str(t.id): t for t in QuerySet(Tool).filter(id__in=tool_ids, tool_type=ToolType.CUSTOM, is_active=True) - } - for tool_id in tool_ids: - tool = custom_tools_map.get(str(tool_id)) - if tool is None: - continue - executor = ToolExecutor() - init_params_default_value = {i["field"]: i.get("default_value") for i in tool.init_field_list} - if tool.init_params is not None: - tool_init_params = init_params_default_value | json.loads(rsa_long_decrypt(tool.init_params)) - else: - tool_init_params = init_params_default_value - - tool_config = executor.get_tool_mcp_config(tool, tool_init_params) - - mcp_servers_config[str(tool.id)] = tool_config - - if application_ids and len(application_ids) > 0: - self.context["application_ids"] = application_ids - apps_map = {str(a.id): a for a in QuerySet(Application).filter(id__in=application_ids, is_publish=True)} - app_keys_map = { - str(ak.application_id): ak - for ak in QuerySet(ApplicationApiKey).filter(application_id__in=application_ids, is_active=True) - } - app_access_tokens_map = { - str(at.application_id): at - for at in QuerySet(ApplicationAccessToken).filter(application_id__in=application_ids) - } - for application_id in application_ids: - app = apps_map.get(str(application_id)) - if app is None: - continue - app_key = app_keys_map.get(str(application_id)) - if app_key is not None: - api_key = app_key.secret_key - application_access_token = app_access_tokens_map.get(str(app_key.application_id)) - if application_access_token is not None and application_access_token.authentication: - raise AppApiException( - 500, - _("Agent 【{name}】 access token authentication is not supported for agent tool").format( - name=app.name - ), - ) - else: - raise AppApiException( - 500, _("Agent Key is required for agent tool 【{name}】").format(name=app.name) - ) - executor = ToolExecutor() - app_config = executor.get_app_mcp_config(api_key, self.get_chat_files(), self.get_form_data()) - mcp_servers_config[app.name] = app_config - - if skill_tool_ids and len(skill_tool_ids) > 0: - self.context["skill_tool_ids"] = skill_tool_ids - skill_file_items = [] - skill_tools_map = {str(t.id): t for t in QuerySet(Tool).filter(id__in=skill_tool_ids, is_active=True)} - for tool_id in skill_tool_ids: - tool = skill_tools_map.get(str(tool_id)) - if tool is None: - continue - init_params_default_value = {i["field"]: i.get("default_value") for i in tool.init_field_list} - if tool.init_params is not None: - params = init_params_default_value | json.loads(rsa_long_decrypt(tool.init_params)) - else: - params = init_params_default_value - - skill_file_items.append({"tool_id": str(tool.id), "file_id": tool.code, "params": params}) - mcp_servers_config["skills"] = skill_file_items - - if len(mcp_servers_config) > 0 or len(tools) > 0: - # 安全获取 application - application_id = None - tool_id = None - knowledge_id = None - if [WorkflowMode.KNOWLEDGE, WorkflowMode.KNOWLEDGE_LOOP].__contains__( - self.workflow_manage.flow.workflow_mode - ): - knowledge_id = self.workflow_params.get("knowledge_id") - elif [WorkflowMode.APPLICATION, WorkflowMode.APPLICATION_LOOP].__contains__( - self.workflow_manage.flow.workflow_mode - ): - application_id = self.workflow_manage.work_flow_post_handler.chat_info.application.id - elif [WorkflowMode.TOOL, WorkflowMode.TOOL_LOOP].__contains__(self.workflow_manage.flow.workflow_mode): - tool_id = self.workflow_params.get("tool_id") - - source_id = application_id or knowledge_id or tool_id - source_type = "APPLICATION" if application_id else "KNOWLEDGE" if knowledge_id else "TOOL" - r = mcp_response_generator( - chat_model, - system_prompt, - message_list, - mcp_servers_config, - mcp_output_enable, - tool_init_params, - source_id, - source_type, - chat_id, - tools, - ) - return NodeResult( - { - "result": r, - "chat_model": chat_model, - "message_list": message_list, - "history_message": [ - {"content": message.content, "role": message.type} - for message in (history_message if history_message is not None else []) - ], - "question": question.content, - }, - {}, - _write_context=write_context_stream, - ) - - return None - - def get_chat_files(self): - """ - 获取本次对话上传的文件, 用于透传给被当作工具调用的应用/MCP - """ - chat_files = {} - for field in CHAT_FILE_LIST_FIELDS: - file_list = getattr(self.workflow_manage, field, None) or [] - items = [ - {key: item.get(key) for key in ("name", "url", "file_id") if item.get(key) is not None} - for item in file_list - if isinstance(item, dict) - ] - if items: - chat_files[field] = items - return chat_files - - def get_form_data(self): - """ - 获取当前会话的用户输入参数，用于透传给作为工具调用的子智能体。 - - 循环工作流会创建独立的 WorkflowManage，并将自身的 form_data 初始化为 - 空字典，因此需要继续从父工作流中查找原始用户输入。 - """ - workflow_manage = self.workflow_manage - visited = set() - while workflow_manage is not None and id(workflow_manage) not in visited: - visited.add(id(workflow_manage)) - form_data = getattr(workflow_manage, "form_data", None) - if isinstance(form_data, dict) and form_data: - return form_data.copy() - workflow_manage = getattr(workflow_manage, "parentWorkflowManage", None) - return {} - - def handle_variables(self, tool_params): - # 处理参数中的变量 - for k, v in tool_params.items(): - if type(v) == str: - tool_params[k] = self.workflow_manage.generate_prompt(tool_params[k]) - elif type(v) == dict: - self.handle_variables(v) - elif (type(v) == list) and len(v) > 0 and (type(v[0]) == str): - tool_params[k] = self.get_reference_content(v) - return tool_params - - def get_reference_content(self, fields: List[str]): - return str(self.workflow_manage.get_reference_field(fields[0], fields[1:])) if fields else "" - - @staticmethod - def get_history_message(history_chat_record, dialogue_number, dialogue_type, runtime_node_id): - start_index = len(history_chat_record) - dialogue_number - history_message = reduce( - lambda x, y: [*x, *y], - [ - get_message(history_chat_record[index], dialogue_type, runtime_node_id) - for index in range(start_index if start_index > 0 else 0, len(history_chat_record)) - ], - [], - ) - for message in history_message: - if isinstance(message.content, str): - message.content = re.sub(r".*?", "", message.content, flags=re.DOTALL) - return history_message - - def generate_prompt_question(self, prompt, model): - image = self.get_image() - video = self.get_video() - vision = self.is_vision() - videos = [] - images = [] - if image and vision: - images = self._process_images(image) - if video and vision: - videos = self._process_videos(video, model) - prompt = self.workflow_manage.generate_prompt(prompt) - if images or videos: - return HumanMessage(content=[*videos, *images, {"type": "text", "text": prompt}]) - return HumanMessage(content=prompt) - - def is_vision(self): - if "vision" in self.node_params_serializer.data: - return self.node_params_serializer.data.get("vision") - return False - - def get_image(self): - if "image_list" in self.node_params_serializer.data: - image = self.workflow_manage.get_reference_field( - self.node_params_serializer.data.get("image_list")[0], - self.node_params_serializer.data.get("image_list")[1:], - ) - return image - return None - - def get_video(self): - if "video_list" in self.node_params_serializer.data: - video = self.workflow_manage.get_reference_field( - self.node_params_serializer.data.get("video_list")[0], - self.node_params_serializer.data.get("video_list")[1:], - ) - return video - return None - - def _process_videos(self, image, video_model): - videos = [] - if isinstance(image, str) and image.startswith("http"): - videos.append({"type": "video_url", "video_url": {"url": image}}) - elif image is not None and len(image) > 0: - for img in image: - if "file_id" in img: - file_id = img["file_id"] - file = QuerySet(File).filter(id=file_id).first() - url = video_model.upload_file_and_get_url(file.get_bytes(), file.file_name) - videos.append({"type": "video_url", "video_url": {"url": url}}) - elif "url" in img and img["url"].startswith("http"): - videos.append({"type": "video_url", "video_url": {"url": img["url"]}}) - return videos - - def _process_images(self, image): - """ - 处理图像数据，转换为模型可识别的格式 - """ - images = [] - if isinstance(image, str) and image.startswith("http"): - images.append({"type": "image_url", "image_url": {"url": image}}) - elif image is not None and len(image) > 0: - for img in image: - if "file_id" in img: - file_id = img["file_id"] - file = QuerySet(File).filter(id=file_id).first() - image_bytes = file.get_bytes() - base64_image = base64.b64encode(image_bytes).decode("utf-8") - image_format = guess_image_format(image_bytes) - images.append( - {"type": "image_url", "image_url": {"url": f"data:image/{image_format};base64,{base64_image}"}} - ) - elif "url" in img and img["url"].startswith("http"): - images.append({"type": "image_url", "image_url": {"url": img["url"]}}) - return images - - def generate_message_list(self, question, history_message): - return [*history_message, question] - - @staticmethod - def reset_message_list(message_list: List[BaseMessage], answer_text): - result = [ - {"role": "user" if isinstance(message, HumanMessage) else "ai", "content": message.content} - for message in message_list - ] - result.append({"role": "ai", "content": answer_text}) - return result - - def get_details(self, index: int, **kwargs): - return { - "name": self.node.properties.get("stepName"), - "index": index, - "run_time": self.context.get("run_time"), - "system": self.context.get("system"), - "history_message": self.context.get("history_message"), - "question": self.context.get("question"), - "answer": self.context.get("answer"), - "reasoning_content": self.context.get("reasoning_content"), - "enableException": self.node.properties.get("enableException"), - "type": self.node.type, - "message_tokens": self.context.get("message_tokens"), - "answer_tokens": self.context.get("answer_tokens"), - "status": self.status, - "err_message": self.err_message, - } diff --git a/apps/application/flow/step_node/application_node/__init__.py b/apps/application/flow/step_node/application_node/__init__.py deleted file mode 100644 index d1ea91ca7f8..00000000000 --- a/apps/application/flow/step_node/application_node/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# coding=utf-8 -from .impl import * diff --git a/apps/application/flow/step_node/application_node/i_application_node.py b/apps/application/flow/step_node/application_node/i_application_node.py deleted file mode 100644 index 30cfd8632fc..00000000000 --- a/apps/application/flow/step_node/application_node/i_application_node.py +++ /dev/null @@ -1,106 +0,0 @@ -# coding=utf-8 -from typing import Type - -from rest_framework import serializers - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import INode, NodeResult - -from django.utils.translation import gettext_lazy as _ - -from application.models import ChatSourceChoices - - -class ApplicationNodeSerializer(serializers.Serializer): - application_id = serializers.CharField(required=True, label=_("Application ID")) - question_reference_address = serializers.ListField(required=True, - label=_("User Questions")) - api_input_field_list = serializers.ListField(required=False, label=_("API Input Fields")) - user_input_field_list = serializers.ListField(required=False, - label=_("User Input Fields")) - image_list = serializers.ListField(required=False, label=_("picture")) - document_list = serializers.ListField(required=False, label=_("document")) - audio_list = serializers.ListField(required=False, label=_("Audio")) - video_list = serializers.ListField(required=False, label=_("Video")) - child_node = serializers.DictField(required=False, allow_null=True, - label=_("Child Nodes")) - node_data = serializers.DictField(required=False, allow_null=True, label=_("Form Data")) - - -class IApplicationNode(INode): - type = 'application-node' - support = [WorkflowMode.APPLICATION, WorkflowMode.APPLICATION_LOOP] - - def get_node_params_serializer_class(self) -> Type[serializers.Serializer]: - return ApplicationNodeSerializer - - def _run(self): - question = self.workflow_manage.get_reference_field( - self.node_params_serializer.data.get('question_reference_address')[0], - self.node_params_serializer.data.get('question_reference_address')[1:]) - kwargs = {} - for api_input_field in self.node_params_serializer.data.get('api_input_field_list', []): - value = api_input_field.get('value', [''])[0] if api_input_field.get('value') else '' - kwargs[api_input_field['variable']] = self.workflow_manage.get_reference_field(value, - api_input_field['value'][ - 1:]) if value != '' else '' - - for user_input_field in self.node_params_serializer.data.get('user_input_field_list', []): - value = user_input_field.get('value', [''])[0] if user_input_field.get('value') else '' - kwargs[user_input_field['field']] = self.workflow_manage.get_reference_field(value, - user_input_field['value'][ - 1:]) if value != '' else '' - # 判断是否包含这个属性 - app_document_list = self.node_params_serializer.data.get('document_list', []) - if app_document_list and len(app_document_list) > 0: - app_document_list = self.workflow_manage.get_reference_field( - app_document_list[0], - app_document_list[1:]) - for document in app_document_list: - if 'file_id' not in document: - raise ValueError( - _("Parameter value error: The uploaded document lacks file_id, and the document upload fails")) - app_image_list = self.node_params_serializer.data.get('image_list', []) - if app_image_list and len(app_image_list) > 0: - app_image_list = self.workflow_manage.get_reference_field( - app_image_list[0], - app_image_list[1:]) - for image in app_image_list: - if 'file_id' not in image: - raise ValueError( - _("Parameter value error: The uploaded image lacks file_id, and the image upload fails")) - - app_audio_list = self.node_params_serializer.data.get('audio_list', []) - if app_audio_list and len(app_audio_list) > 0: - app_audio_list = self.workflow_manage.get_reference_field( - app_audio_list[0], - app_audio_list[1:]) - for audio in app_audio_list: - if 'file_id' not in audio: - raise ValueError( - _("Parameter value error: The uploaded audio lacks file_id, and the audio upload fails.")) - app_video_list = self.node_params_serializer.data.get('video_list', []) - if app_video_list and len(app_video_list) > 0: - app_video_list = self.workflow_manage.get_reference_field( - app_video_list[0], - app_video_list[1:] - ) - for video in app_video_list: - if 'file_id' not in video: - raise ValueError( - _("Parameter value error: The uploaded video lacks file_id, and the video upload fails.")) - return self.execute(**{**self.flow_params_serializer.data, **self.node_params_serializer.data}, - app_document_list=app_document_list, app_image_list=app_image_list, - app_audio_list=app_audio_list, - app_video_list=app_video_list, - ip_address=self.workflow_params.get('ip_address') or '-', - source=self.workflow_params.get('source') or {"type": ChatSourceChoices.ONLINE.value}, - message=str(question), **kwargs) - - def execute(self, application_id, message, chat_id, chat_record_id, stream, re_chat, client_id, client_type, - app_document_list=None, app_image_list=None, app_audio_list=None, app_video_list=None, child_node=None, - node_data=None, - ip_address=None, - source=None, - **kwargs) -> NodeResult: - pass diff --git a/apps/application/flow/step_node/application_node/impl/__init__.py b/apps/application/flow/step_node/application_node/impl/__init__.py deleted file mode 100644 index e31a8d885cd..00000000000 --- a/apps/application/flow/step_node/application_node/impl/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# coding=utf-8 -from .base_application_node import BaseApplicationNode diff --git a/apps/application/flow/step_node/application_node/impl/base_application_node.py b/apps/application/flow/step_node/application_node/impl/base_application_node.py deleted file mode 100644 index 1288facc819..00000000000 --- a/apps/application/flow/step_node/application_node/impl/base_application_node.py +++ /dev/null @@ -1,299 +0,0 @@ -# coding=utf-8 -import json -import re -import time -import uuid -from typing import Dict, List -from django.utils.translation import gettext as _ -from application.flow.common import Answer -from application.flow.i_step_node import NodeResult, INode -from application.flow.step_node.application_node.i_application_node import IApplicationNode -from common.utils.logger import maxkb_logger -from application.models import Chat, ChatSourceChoices - - -def string_to_uuid(input_str): - return str(uuid.uuid5(uuid.NAMESPACE_DNS, input_str)) - - -def _is_interrupt_exec(node, node_variable: Dict, workflow_variable: Dict): - return node_variable.get('is_interrupt_exec', False) - - -def _write_context(node_variable: Dict, workflow_variable: Dict, node: INode, workflow, answer: str, - reasoning_content: str): - result = node_variable.get('result') - node.context['application_node_dict'] = node_variable.get('application_node_dict') - node.context['node_dict'] = node_variable.get('node_dict', {}) - node.context['is_interrupt_exec'] = node_variable.get('is_interrupt_exec') - node.context['message_tokens'] = result.get('usage', {}).get('prompt_tokens', 0) - node.context['answer_tokens'] = result.get('usage', {}).get('completion_tokens', 0) - node.context['answer'] = answer - node.context['result'] = answer - node.context['reasoning_content'] = reasoning_content - node.context['question'] = node_variable['question'] - node.context['run_time'] = time.time() - node.context['start_time'] - if workflow.is_result(node, NodeResult(node_variable, workflow_variable)): - node.answer_text = answer - - -def write_context_stream(node_variable: Dict, workflow_variable: Dict, node: INode, workflow): - """ - 写入上下文数据 (流式) - @param node_variable: 节点数据 - @param workflow_variable: 全局数据 - @param node: 节点 - @param workflow: 工作流管理器 - """ - response = node_variable.get('result') - answer = '' - reasoning_content = '' - usage = {} - node_child_node = {} - application_node_dict = node.context.get('application_node_dict', {}) - is_interrupt_exec = False - for chunk in response: - # 先把流转成字符串 - response_content = chunk.decode('utf-8')[6:] - response_content = json.loads(response_content) - content = (response_content.get('content', '') or '') - runtime_node_id = response_content.get('runtime_node_id', '') - chat_record_id = response_content.get('chat_record_id', '') - child_node = response_content.get('child_node') - view_type = response_content.get('view_type') - node_type = response_content.get('node_type') - real_node_id = response_content.get('real_node_id') - node_is_end = response_content.get('node_is_end', False) - _reasoning_content = (response_content.get('reasoning_content', '') or '') - if node_type == 'form-node': - is_interrupt_exec = True - answer += content - reasoning_content += _reasoning_content - node_child_node = {'runtime_node_id': runtime_node_id, 'chat_record_id': chat_record_id, - 'child_node': child_node} - - if real_node_id is not None: - real_node_id = real_node_id + '__' + node.runtime_node_id - application_node = application_node_dict.get(real_node_id, None) - if application_node is None: - - application_node_dict[real_node_id] = {'content': content, - 'runtime_node_id': runtime_node_id, - 'chat_record_id': chat_record_id, - 'child_node': child_node, - 'index': len(application_node_dict), - 'view_type': view_type, - 'reasoning_content': _reasoning_content} - else: - application_node['content'] += content - application_node['reasoning_content'] += _reasoning_content - - yield {'content': content, - 'node_type': node_type, - 'runtime_node_id': runtime_node_id, 'chat_record_id': chat_record_id, - 'reasoning_content': _reasoning_content, - 'child_node': child_node, - 'real_node_id': real_node_id, - 'node_is_end': node_is_end, - 'view_type': view_type} - usage = response_content.get('usage', {}) - node_variable['result'] = {'usage': usage} - node_variable['is_interrupt_exec'] = is_interrupt_exec - node_variable['child_node'] = node_child_node - node_variable['application_node_dict'] = application_node_dict - _write_context(node_variable, workflow_variable, node, workflow, answer, reasoning_content) - - -def write_context(node_variable: Dict, workflow_variable: Dict, node: INode, workflow): - """ - 写入上下文数据 - @param node_variable: 节点数据 - @param workflow_variable: 全局数据 - @param node: 节点实例对象 - @param workflow: 工作流管理器 - """ - response = node_variable.get('result', {}).get('data', {}) - node_variable['result'] = {'usage': {'completion_tokens': response.get('completion_tokens'), - 'prompt_tokens': response.get('prompt_tokens')}} - answer = response.get('content', '') or "抱歉，没有查找到相关内容，请重新描述您的问题或提供更多信息。" - reasoning_content = response.get('reasoning_content', '') - answer_list = response.get('answer_list', []) - node_variable['application_node_dict'] = {answer.get('real_node_id'): {**answer, 'index': index} for answer, index - in - zip(answer_list, range(len(answer_list)))} - _write_context(node_variable, workflow_variable, node, workflow, answer, reasoning_content) - - -def reset_application_node_dict(application_node_dict, runtime_node_id, node_data): - try: - if application_node_dict is None: - return - for key in application_node_dict: - application_node = application_node_dict[key] - if application_node.get('runtime_node_id') == runtime_node_id: - content: str = application_node.get('content') - match = re.search(r'.*?', content, flags=re.DOTALL) - if match: - form_setting_str = match.group().replace('', '').replace('', '') - form_setting = json.loads(form_setting_str) - form_setting['is_submit'] = True - form_setting['form_data'] = node_data - value = f'{json.dumps(form_setting)}' - res = re.sub(r'.*?', '${value}', content, flags=re.DOTALL) - application_node['content'] = res.replace('${value}', value) - except Exception as e: - maxkb_logger.warning(f'reset_application_node_dict error: {e}', exc_info=True) - - -class BaseApplicationNode(IApplicationNode): - def get_answer_list(self) -> List[Answer] | None: - if self.answer_text is None: - return None - application_node_dict = self.context.get('application_node_dict') - if application_node_dict is None or len(application_node_dict) == 0: - return [ - Answer(self.answer_text, self.view_type, self.runtime_node_id, self.workflow_params['chat_record_id'], - self.context.get('child_node'), self.runtime_node_id, '')] - else: - return [Answer(n.get('content'), n.get('view_type'), self.runtime_node_id, - self.workflow_params['chat_record_id'], {'runtime_node_id': n.get('runtime_node_id'), - 'chat_record_id': n.get('chat_record_id') - , 'child_node': n.get('child_node')}, n.get('real_node_id'), - n.get('reasoning_content', '')) - for n in - sorted(application_node_dict.values(), key=lambda item: item.get('index'))] - - def save_context(self, details, workflow_manage): - self.context['answer'] = details.get('answer') - self.context['result'] = details.get('answer') - self.context['question'] = details.get('question') - self.context['type'] = details.get('type') - self.context['reasoning_content'] = details.get('reasoning_content') - self.context['exception_message'] = details.get('err_message') - if self.node_params.get('is_result', False): - self.answer_text = details.get('answer') - - def get_chat_asker(self, kwargs): - asker = kwargs.get('asker') - if asker: - if isinstance(asker, dict): - return asker - return {'username': asker} - return self.workflow_manage.work_flow_post_handler.chat_info.get_chat_user() - - def execute(self, application_id, message, chat_id, chat_record_id, stream, re_chat, - chat_user_id, - chat_user_type, - app_document_list=None, app_image_list=None, app_audio_list=None, app_video_list=None, child_node=None, - node_data=None, - ip_address=None, - source=None, - **kwargs) -> NodeResult: - from chat.serializers.chat import ChatSerializers - if application_id == self.workflow_manage.get_body().get('application_id'): - raise Exception(_("The sub application cannot use the current node")) - # 生成嵌入应用的chat_id - current_chat_id = string_to_uuid(chat_id + application_id) - Chat.objects.get_or_create(id=current_chat_id, defaults={ - 'application_id': application_id, - 'abstract': message[0:1024], - 'chat_user_id': chat_user_id, - 'chat_user_type': chat_user_type, - 'ip_address': ip_address, - 'source': source, - 'asker': self.get_chat_asker(kwargs) - }) - if app_document_list is None: - app_document_list = [] - if app_image_list is None: - app_image_list = [] - if app_audio_list is None: - app_audio_list = [] - if app_video_list is None: - app_video_list = [] - runtime_node_id = None - record_id = None - child_node_value = None - if child_node is not None: - runtime_node_id = child_node.get('runtime_node_id') - record_id = child_node.get('chat_record_id') - child_node_value = child_node.get('child_node') - application_node_dict = self.context.get('application_node_dict') - reset_application_node_dict(application_node_dict, runtime_node_id, node_data) - response = ChatSerializers(data={ - "chat_id": current_chat_id, - "chat_user_id": chat_user_id, - 'chat_user_type': chat_user_type, - 'application_id': application_id, - 'ip_address': ip_address, - 'source': source, - 'debug': False - }).chat(instance= - {'message': message, - 're_chat': re_chat, - 'stream': stream, - 'document_list': [*app_document_list], - 'image_list': [*app_image_list], - 'audio_list': [*app_audio_list], - 'video_list': [*app_video_list], - 'runtime_node_id': runtime_node_id, - 'chat_record_id': record_id, - 'child_node': child_node_value, - 'node_data': node_data, - 'form_data': kwargs} - ) - - if response.status_code == 200: - if stream: - content_generator = response.streaming_content - return NodeResult({'result': content_generator, 'question': message}, {}, - _write_context=write_context_stream, _is_interrupt=_is_interrupt_exec) - else: - data = json.loads(response.content) - return NodeResult({'result': data, 'question': message}, {}, - _write_context=write_context, _is_interrupt=_is_interrupt_exec) - - def get_details(self, index: int, **kwargs): - global_fields = [] - for api_input_field in self.node_params_serializer.data.get('api_input_field_list', []): - value = api_input_field.get('value', [''])[0] if api_input_field.get('value') else '' - global_fields.append({ - 'label': api_input_field['variable'], - 'key': api_input_field['variable'], - 'value': self.workflow_manage.get_reference_field( - value, - api_input_field['value'][1:] - ) if value != '' else '' - }) - - for user_input_field in self.node_params_serializer.data.get('user_input_field_list', []): - value = user_input_field.get('value', [''])[0] if user_input_field.get('value') else '' - global_fields.append({ - 'label': user_input_field['label'], - 'key': user_input_field['field'], - 'value': self.workflow_manage.get_reference_field( - value, - user_input_field['value'][1:] - ) if value != '' else '' - }) - return { - 'name': self.node.properties.get('stepName'), - "index": index, - "info": self.node.properties.get('node_data'), - 'run_time': self.context.get('run_time'), - 'question': self.context.get('question'), - 'answer': self.context.get('answer'), - 'reasoning_content': self.context.get('reasoning_content'), - 'type': self.node.type, - 'message_tokens': self.context.get('message_tokens'), - 'answer_tokens': self.context.get('answer_tokens'), - 'status': self.status, - 'err_message': self.err_message, - 'global_fields': global_fields, - 'document_list': self.workflow_manage.document_list, - 'image_list': self.workflow_manage.image_list, - 'audio_list': self.workflow_manage.audio_list, - 'video_list': self.workflow_manage.video_list, - 'application_node_dict': self.context.get('application_node_dict'), - 'enableException': self.node.properties.get('enableException'), - } diff --git a/apps/application/flow/step_node/condition_node/__init__.py b/apps/application/flow/step_node/condition_node/__init__.py deleted file mode 100644 index 57638504c9e..00000000000 --- a/apps/application/flow/step_node/condition_node/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： __init__.py.py - @date：2024/6/7 14:43 - @desc: -""" -from .impl import * diff --git a/apps/application/flow/step_node/condition_node/i_condition_node.py b/apps/application/flow/step_node/condition_node/i_condition_node.py deleted file mode 100644 index 664ee91baff..00000000000 --- a/apps/application/flow/step_node/condition_node/i_condition_node.py +++ /dev/null @@ -1,42 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： i_condition_node.py - @date：2024/6/7 9:54 - @desc: -""" -from typing import Type - -from django.utils.translation import gettext_lazy as _ -from rest_framework import serializers - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import INode - - -class ConditionSerializer(serializers.Serializer): - compare = serializers.CharField(required=True, label=_("Comparator")) - value = serializers.CharField(required=True, label=_("value")) - field = serializers.ListField(required=True, label=_("Fields")) - - -class ConditionBranchSerializer(serializers.Serializer): - id = serializers.CharField(required=True, label=_("Branch id")) - type = serializers.CharField(required=True, label=_("Branch Type")) - condition = serializers.CharField(required=True, label=_("Condition or|and")) - conditions = ConditionSerializer(many=True) - - -class ConditionNodeParamsSerializer(serializers.Serializer): - branch = ConditionBranchSerializer(many=True) - - -class IConditionNode(INode): - def get_node_params_serializer_class(self) -> Type[serializers.Serializer]: - return ConditionNodeParamsSerializer - - type = 'condition-node' - - support = [WorkflowMode.APPLICATION, WorkflowMode.APPLICATION_LOOP, WorkflowMode.KNOWLEDGE, - WorkflowMode.KNOWLEDGE_LOOP, WorkflowMode.TOOL, WorkflowMode.TOOL_LOOP] diff --git a/apps/application/flow/step_node/condition_node/impl/__init__.py b/apps/application/flow/step_node/condition_node/impl/__init__.py deleted file mode 100644 index c21cd3ebb37..00000000000 --- a/apps/application/flow/step_node/condition_node/impl/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： __init__.py - @date：2024/6/11 15:35 - @desc: -""" -from .base_condition_node import BaseConditionNode diff --git a/apps/application/flow/step_node/condition_node/impl/base_condition_node.py b/apps/application/flow/step_node/condition_node/impl/base_condition_node.py deleted file mode 100644 index e0da03ace4c..00000000000 --- a/apps/application/flow/step_node/condition_node/impl/base_condition_node.py +++ /dev/null @@ -1,47 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： base_condition_node.py - @date：2024/6/7 11:29 - @desc: -""" -from typing import List - -from application.flow.i_step_node import NodeResult -from application.flow.compare import do_assertion -from application.flow.step_node.condition_node.i_condition_node import IConditionNode - - -class BaseConditionNode(IConditionNode): - def save_context(self, details, workflow_manage): - self.context['branch_id'] = details.get('branch_id') - self.context['branch_name'] = details.get('branch_name') - self.context['exception_message'] = details.get('err_message') - - def execute(self, **kwargs) -> NodeResult: - branch_list = self.node_params_serializer.data['branch'] - branch = self._execute(branch_list) - r = NodeResult({'branch_id': branch.get('id'), 'branch_name': branch.get('type')}, {}) - return r - - def _execute(self, branch_list: List): - for branch in branch_list: - if self.branch_assertion(branch): - return branch - - def branch_assertion(self, branch): - return do_assertion(self.workflow_manage, branch.get('condition'), branch.get('conditions')) - - def get_details(self, index: int, **kwargs): - return { - 'name': self.node.properties.get('stepName'), - "index": index, - 'run_time': self.context.get('run_time'), - 'branch_id': self.context.get('branch_id'), - 'branch_name': self.context.get('branch_name'), - 'type': self.node.type, - 'status': self.status, - 'err_message': self.err_message, - 'enableException': self.node.properties.get('enableException'), - } diff --git a/apps/application/flow/step_node/data_source_local_node/__init__.py b/apps/application/flow/step_node/data_source_local_node/__init__.py deleted file mode 100644 index bbf804a7079..00000000000 --- a/apps/application/flow/step_node/data_source_local_node/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎虎 - @file： __init__.py.py - @date：2025/11/11 10:06 - @desc: -""" diff --git a/apps/application/flow/step_node/data_source_local_node/i_data_source_local_node.py b/apps/application/flow/step_node/data_source_local_node/i_data_source_local_node.py deleted file mode 100644 index e6b39f686fa..00000000000 --- a/apps/application/flow/step_node/data_source_local_node/i_data_source_local_node.py +++ /dev/null @@ -1,42 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎虎 - @file： i_data_source_local_node.py - @date：2025/11/11 10:06 - @desc: -""" -from abc import abstractmethod -from typing import Type - -from django.utils.translation import gettext_lazy as _ -from rest_framework import serializers - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import INode, NodeResult - - -class DataSourceLocalNodeParamsSerializer(serializers.Serializer): - file_type_list = serializers.ListField(child=serializers.CharField(label=('')), label='') - file_size_limit = serializers.IntegerField(required=True, label=_("Number of uploaded files")) - file_count_limit = serializers.IntegerField(required=True, label=_("Upload file size")) - - -class IDataSourceLocalNode(INode): - type = 'data-source-local-node' - - @staticmethod - @abstractmethod - def get_form_list(node): - pass - - def get_node_params_serializer_class(self) -> Type[serializers.Serializer]: - return DataSourceLocalNodeParamsSerializer - - def _run(self): - return self.execute(**self.node_params_serializer.data, **self.flow_params_serializer.data) - - def execute(self, file_type_list, file_size_limit, file_count_limit, **kwargs) -> NodeResult: - pass - - support = [WorkflowMode.KNOWLEDGE] diff --git a/apps/application/flow/step_node/data_source_local_node/impl/__init__.py b/apps/application/flow/step_node/data_source_local_node/impl/__init__.py deleted file mode 100644 index 6f830151971..00000000000 --- a/apps/application/flow/step_node/data_source_local_node/impl/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎虎 - @file： __init__.py.py - @date：2025/11/11 10:08 - @desc: -""" diff --git a/apps/application/flow/step_node/data_source_local_node/impl/base_data_source_local_node.py b/apps/application/flow/step_node/data_source_local_node/impl/base_data_source_local_node.py deleted file mode 100644 index c2f69b6f21a..00000000000 --- a/apps/application/flow/step_node/data_source_local_node/impl/base_data_source_local_node.py +++ /dev/null @@ -1,52 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎虎 - @file： base_data_source_local_node.py - @date：2025/11/11 10:30 - @desc: -""" -from application.flow.i_step_node import NodeResult -from application.flow.step_node.data_source_local_node.i_data_source_local_node import IDataSourceLocalNode -from common import forms -from common.forms import BaseForm - - -class BaseDataSourceLocalNodeForm(BaseForm): - api_key = forms.PasswordInputField('API Key', required=True) - - -class BaseDataSourceLocalNode(IDataSourceLocalNode): - def save_context(self, details, workflow_manage): - self.context['exception_message'] = details.get('err_message') - - @staticmethod - def get_form_list(node): - node_data = node.get('properties').get('node_data') - return [{ - 'field': 'file_list', - 'input_type': 'LocalFileUpload', - 'attrs': { - 'file_count_limit': node_data.get('file_count_limit') or 10, - 'file_size_limit': node_data.get('file_size_limit') or 100, - 'file_type_list': node_data.get('file_type_list'), - }, - 'label': '', - }] - - def execute(self, file_type_list, file_size_limit, file_count_limit, **kwargs) -> NodeResult: - return NodeResult({'file_list': self.workflow_manage.params.get('data_source', {}).get('file_list')}, - self.workflow_manage.params.get('knowledge_base') or {}) - - def get_details(self, index: int, **kwargs): - return { - 'name': self.node.properties.get('stepName'), - "index": index, - 'run_time': self.context.get('run_time'), - 'type': self.node.type, - 'file_list': self.context.get('file_list'), - 'knowledge_base': self.workflow_params.get('knowledge_base'), - 'status': self.status, - 'err_message': self.err_message, - 'enableException': self.node.properties.get('enableException'), - } diff --git a/apps/application/flow/step_node/data_source_web_node/__init__.py b/apps/application/flow/step_node/data_source_web_node/__init__.py deleted file mode 100644 index 461bab6fc12..00000000000 --- a/apps/application/flow/step_node/data_source_web_node/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：niu - @file： __init__.py.py - @date：2025/11/12 13:43 - @desc: -""" diff --git a/apps/application/flow/step_node/data_source_web_node/i_data_source_web_node.py b/apps/application/flow/step_node/data_source_web_node/i_data_source_web_node.py deleted file mode 100644 index ee5dc990b84..00000000000 --- a/apps/application/flow/step_node/data_source_web_node/i_data_source_web_node.py +++ /dev/null @@ -1,28 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：niu - @file： i_data_source_web_node.py - @date：2025/11/12 13:47 - @desc: -""" -from abc import abstractmethod - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import INode, NodeResult - - -class IDataSourceWebNode(INode): - type = 'data-source-web-node' - support = [WorkflowMode.KNOWLEDGE] - - @staticmethod - @abstractmethod - def get_form_list(node): - pass - - def _run(self): - return self.execute(**self.flow_params_serializer.data) - - def execute(self, **kwargs) -> NodeResult: - pass diff --git a/apps/application/flow/step_node/data_source_web_node/impl/__init__.py b/apps/application/flow/step_node/data_source_web_node/impl/__init__.py deleted file mode 100644 index b7541b12df1..00000000000 --- a/apps/application/flow/step_node/data_source_web_node/impl/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：niu - @file： __init__.py - @date：2025/11/12 13:44 - @desc: -""" diff --git a/apps/application/flow/step_node/data_source_web_node/impl/base_data_source_web_node.py b/apps/application/flow/step_node/data_source_web_node/impl/base_data_source_web_node.py deleted file mode 100644 index 0a9ec336036..00000000000 --- a/apps/application/flow/step_node/data_source_web_node/impl/base_data_source_web_node.py +++ /dev/null @@ -1,98 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：niu - @file： base_data_source_web_node.py - @date：2025/11/12 13:47 - @desc: -""" -import traceback - -from django.utils.translation import gettext_lazy as _ - -from application.flow.i_step_node import NodeResult -from application.flow.step_node.data_source_web_node.i_data_source_web_node import IDataSourceWebNode -from common import forms -from common.forms import BaseForm -from common.utils.fork import ForkManage, Fork, ChildLink -from common.utils.logger import maxkb_logger - - -class BaseDataSourceWebNodeForm(BaseForm): - source_url = forms.TextInputField(_('Web source url'), required=True, attrs={ - 'placeholder': _('Please enter the Web root address')}) - selector = forms.TextInputField(_('Web knowledge selector'), required=False, attrs={ - 'placeholder': _('The default is body, you can enter .classname/#idname/tagname')}) - - -class InterruptedTaskException(Exception): - def __init__(self, *args, **kwargs): # real signature unknown - pass - - -def get_collect_handler(workflow_manage): - results = [] - - def handler(child_link: ChildLink, response: Fork.Response): - if response.status == 200: - try: - document_name = child_link.tag.text if child_link.tag is not None and len( - child_link.tag.text.strip()) > 0 else child_link.url - results.append({ - "name": document_name.strip(), - "content": response.content, - }) - - except Exception as e: - maxkb_logger.error(f'{str(e)}:{traceback.format_exc()}') - if workflow_manage.is_the_task_interrupted(): - raise InterruptedTaskException('Task interrupted') - - return handler, results - - -class BaseDataSourceWebNode(IDataSourceWebNode): - def save_context(self, details, workflow_manage): - self.context['exception_message'] = details.get('err_message') - - @staticmethod - def get_form_list(node): - return BaseDataSourceWebNodeForm().to_form_list() - - def execute(self, **kwargs) -> NodeResult: - BaseDataSourceWebNodeForm().valid_form(self.workflow_params.get("data_source")) - - data_source = self.workflow_params.get("data_source") - - node_id = data_source.get("node_id") - source_url = data_source.get("source_url") - selector = data_source.get("selector") or "body" - - collect_handler, document_list = get_collect_handler(self.workflow_manage) - - try: - ForkManage(source_url, selector.split(" ") if selector is not None else []).fork(3, set(), collect_handler) - - return NodeResult({'document_list': document_list, 'source_url': source_url, 'selector': selector}, - self.workflow_manage.params.get('knowledge_base') or {}) - - except Exception as e: - if isinstance(e, InterruptedTaskException): - return NodeResult({'document_list': document_list, 'source_url': source_url, 'selector': selector}, - self.workflow_manage.params.get('knowledge_base') or {}) - maxkb_logger.error(_('data source web node:{node_id} error{error}{traceback}').format( - node_id=node_id, error=str(e), traceback=traceback.format_exc())) - - def get_details(self, index: int, **kwargs): - return { - 'name': self.node.properties.get('stepName'), - "index": index, - 'run_time': self.context.get('run_time'), - 'type': self.node.type, - 'input_params': {"source_url": self.context.get("source_url"), "selector": self.context.get('selector')}, - 'output_params': self.context.get('document_list'), - 'knowledge_base': self.workflow_params.get('knowledge_base'), - 'status': self.status, - 'err_message': self.err_message, - 'enableException': self.node.properties.get('enableException'), - } diff --git a/apps/application/flow/step_node/direct_reply_node/__init__.py b/apps/application/flow/step_node/direct_reply_node/__init__.py deleted file mode 100644 index cf360f95685..00000000000 --- a/apps/application/flow/step_node/direct_reply_node/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： __init__.py - @date：2024/6/11 17:50 - @desc: -""" -from .impl import * \ No newline at end of file diff --git a/apps/application/flow/step_node/direct_reply_node/i_reply_node.py b/apps/application/flow/step_node/direct_reply_node/i_reply_node.py deleted file mode 100644 index 1a963d76a58..00000000000 --- a/apps/application/flow/step_node/direct_reply_node/i_reply_node.py +++ /dev/null @@ -1,58 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： i_reply_node.py - @date：2024/6/11 16:25 - @desc: -""" -from typing import Type - -from rest_framework import serializers - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import INode, NodeResult -from common.exception.app_exception import AppApiException - -from django.utils.translation import gettext_lazy as _ - - -class ReplyNodeParamsSerializer(serializers.Serializer): - reply_type = serializers.CharField(required=True, label=_("Response Type")) - fields = serializers.ListField(required=False, label=_("Reference Field")) - content = serializers.CharField(required=False, allow_blank=True, allow_null=True, - label=_("Direct answer content")) - is_result = serializers.BooleanField(required=False, - label=_('Whether to return content')) - - def is_valid(self, *, raise_exception=False): - super().is_valid(raise_exception=True) - if self.data.get('reply_type') == 'referencing': - if 'fields' not in self.data: - raise AppApiException(500, _("Reference field cannot be empty")) - if len(self.data.get('fields')) < 2: - raise AppApiException(500, _("Reference field error")) - else: - if 'content' not in self.data or self.data.get('content') is None: - raise AppApiException(500, _("Content cannot be empty")) - - -class IReplyNode(INode): - type = 'reply-node' - support = [WorkflowMode.APPLICATION, WorkflowMode.APPLICATION_LOOP, WorkflowMode.KNOWLEDGE_LOOP, - WorkflowMode.KNOWLEDGE, WorkflowMode.TOOL, WorkflowMode.TOOL_LOOP] - - def get_node_params_serializer_class(self) -> Type[serializers.Serializer]: - return ReplyNodeParamsSerializer - - def _run(self): - if [WorkflowMode.KNOWLEDGE, WorkflowMode.KNOWLEDGE_LOOP, WorkflowMode.TOOL, - WorkflowMode.TOOL_LOOP].__contains__( - self.workflow_manage.flow.workflow_mode): - return self.execute(**self.node_params_serializer.data, **self.flow_params_serializer.data, - **{'stream': True}) - else: - return self.execute(**self.node_params_serializer.data, **self.flow_params_serializer.data) - - def execute(self, reply_type, stream, fields=None, content=None, **kwargs) -> NodeResult: - pass diff --git a/apps/application/flow/step_node/direct_reply_node/impl/__init__.py b/apps/application/flow/step_node/direct_reply_node/impl/__init__.py deleted file mode 100644 index 3307e90899e..00000000000 --- a/apps/application/flow/step_node/direct_reply_node/impl/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： __init__.py - @date：2024/6/11 17:49 - @desc: -""" -from .base_reply_node import * \ No newline at end of file diff --git a/apps/application/flow/step_node/direct_reply_node/impl/base_reply_node.py b/apps/application/flow/step_node/direct_reply_node/impl/base_reply_node.py deleted file mode 100644 index e70c45afd07..00000000000 --- a/apps/application/flow/step_node/direct_reply_node/impl/base_reply_node.py +++ /dev/null @@ -1,47 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： base_reply_node.py - @date：2024/6/11 17:25 - @desc: -""" -from typing import List - -from application.flow.i_step_node import NodeResult -from application.flow.step_node.direct_reply_node.i_reply_node import IReplyNode - - -class BaseReplyNode(IReplyNode): - def save_context(self, details, workflow_manage): - self.context['answer'] = details.get('answer') - self.context['exception_message'] = details.get('err_message') - if self.node_params.get('is_result', False): - self.answer_text = details.get('answer') - - def execute(self, reply_type, stream, fields=None, content=None, **kwargs) -> NodeResult: - if reply_type == 'referencing': - result = self.get_reference_content(fields) - else: - result = self.generate_reply_content(content) - return NodeResult({'answer': result}, {}) - - def generate_reply_content(self, prompt): - return self.workflow_manage.generate_prompt(prompt) - - def get_reference_content(self, fields: List[str]): - return str(self.workflow_manage.get_reference_field( - fields[0], - fields[1:])) if fields else '' - - def get_details(self, index: int, **kwargs): - return { - 'name': self.node.properties.get('stepName'), - "index": index, - 'run_time': self.context.get('run_time'), - 'type': self.node.type, - 'answer': self.context.get('answer'), - 'status': self.status, - 'err_message': self.err_message, - 'enableException': self.node.properties.get('enableException'), - } diff --git a/apps/application/flow/step_node/document_extract_node/__init__.py b/apps/application/flow/step_node/document_extract_node/__init__.py deleted file mode 100644 index ce8f10f3e24..00000000000 --- a/apps/application/flow/step_node/document_extract_node/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .impl import * \ No newline at end of file diff --git a/apps/application/flow/step_node/document_extract_node/i_document_extract_node.py b/apps/application/flow/step_node/document_extract_node/i_document_extract_node.py deleted file mode 100644 index d2cf43e0238..00000000000 --- a/apps/application/flow/step_node/document_extract_node/i_document_extract_node.py +++ /dev/null @@ -1,30 +0,0 @@ -# coding=utf-8 - -from typing import Type - -from django.utils.translation import gettext_lazy as _ -from rest_framework import serializers - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import INode, NodeResult - - -class DocumentExtractNodeSerializer(serializers.Serializer): - document_list = serializers.ListField(required=False, label=_("document")) - - -class IDocumentExtractNode(INode): - type = 'document-extract-node' - support = [WorkflowMode.APPLICATION, WorkflowMode.APPLICATION_LOOP, WorkflowMode.KNOWLEDGE_LOOP, - WorkflowMode.KNOWLEDGE, WorkflowMode.TOOL, WorkflowMode.TOOL_LOOP] - - def get_node_params_serializer_class(self) -> Type[serializers.Serializer]: - return DocumentExtractNodeSerializer - - def _run(self): - res = self.workflow_manage.get_reference_field(self.node_params_serializer.data.get('document_list')[0], - self.node_params_serializer.data.get('document_list')[1:]) - return self.execute(document=res, **self.flow_params_serializer.data) - - def execute(self, document, chat_id=None, **kwargs) -> NodeResult: - pass diff --git a/apps/application/flow/step_node/document_extract_node/impl/__init__.py b/apps/application/flow/step_node/document_extract_node/impl/__init__.py deleted file mode 100644 index cf9d55ecde8..00000000000 --- a/apps/application/flow/step_node/document_extract_node/impl/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .base_document_extract_node import BaseDocumentExtractNode diff --git a/apps/application/flow/step_node/document_extract_node/impl/base_document_extract_node.py b/apps/application/flow/step_node/document_extract_node/impl/base_document_extract_node.py deleted file mode 100644 index b285e90b07d..00000000000 --- a/apps/application/flow/step_node/document_extract_node/impl/base_document_extract_node.py +++ /dev/null @@ -1,95 +0,0 @@ -# coding=utf-8 -import ast -import io - -import uuid_utils.compat as uuid -from django.db.models import QuerySet - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import NodeResult -from application.flow.step_node.document_extract_node.i_document_extract_node import IDocumentExtractNode -from knowledge.models import File, FileSourceType -from knowledge.serializers.document import split_handles, parse_table_handle_list, FileBufferHandle - -splitter = '\n`-----------------------------------`\n' - - -class BaseDocumentExtractNode(IDocumentExtractNode): - def save_context(self, details, workflow_manage): - self.context['content'] = details.get('content') - self.context['exception_message'] = details.get('err_message') - - def execute(self, document, chat_id=None, **kwargs): - get_buffer = FileBufferHandle().get_buffer - - self.context['document_list'] = document - content = [] - if document is None or not isinstance(document, list): - return NodeResult({'content': '', 'document_list': []}, {}) - - # 安全获取 application - application_id = None - tool_id = None - knowledge_id = None - if [WorkflowMode.KNOWLEDGE, WorkflowMode.KNOWLEDGE_LOOP].__contains__(self.workflow_manage.flow.workflow_mode): - knowledge_id = self.workflow_params.get('knowledge_id') - elif [WorkflowMode.APPLICATION, WorkflowMode.APPLICATION_LOOP].__contains__( - self.workflow_manage.flow.workflow_mode): - application_id = self.workflow_manage.work_flow_post_handler.chat_info.application_id - elif [WorkflowMode.TOOL, WorkflowMode.TOOL_LOOP].__contains__(self.workflow_manage.flow.workflow_mode): - tool_id = self.workflow_params.get('tool_id') - - # doc文件中的图片保存 - def save_image(image_list): - for image in image_list: - meta = { - 'debug': False if (application_id or knowledge_id or tool_id) else True, - 'chat_id': chat_id, - 'application_id': str(application_id) if application_id else None, - 'knowledge_id': str(knowledge_id) if knowledge_id else None, - 'tool_id': str(tool_id) if tool_id else None, - 'file_id': str(image.id) - } - file_bytes = image.meta.pop('content') - new_file = File( - id=meta['file_id'], - file_name=image.file_name, - file_size=len(file_bytes), - source_type=FileSourceType.APPLICATION.value if application_id else FileSourceType.KNOWLEDGE.value if knowledge_id else FileSourceType.TOOL.value, - source_id=application_id or knowledge_id or tool_id, - meta=meta - ) - if not QuerySet(File).filter(id=new_file.id).exists(): - new_file.save(file_bytes) - - document_list = [] - for doc in document: - file = QuerySet(File).filter(id=doc['file_id']).first() - buffer = io.BytesIO(file.get_bytes()) - buffer.name = doc['name'] # this is the important line - - for split_handle in (parse_table_handle_list + split_handles): - if split_handle.support(buffer, get_buffer): - # 回到文件头 - buffer.seek(0) - file_content = split_handle.get_content(buffer, save_image) - content.append('### ' + doc['name'] + '\n' + file_content) - document_list.append({'id': str(file.id), 'name': doc['name'], 'content': file_content}) - break - - return NodeResult({'content': splitter.join(content), 'document_list': document_list}, {}) - - def get_details(self, index: int, **kwargs): - content = self.context.get('content', '').split(splitter) - # 不保存content全部内容，因为content内容可能会很大 - return { - 'name': self.node.properties.get('stepName'), - "index": index, - 'run_time': self.context.get('run_time'), - 'type': self.node.type, - 'content': [file_content[:500] for file_content in content], - 'status': self.status, - 'err_message': self.err_message, - 'document_list': self.context.get('document_list'), - 'enableException': self.node.properties.get('enableException'), - } diff --git a/apps/application/flow/step_node/document_split_node/__init__.py b/apps/application/flow/step_node/document_split_node/__init__.py deleted file mode 100644 index ce8f10f3e24..00000000000 --- a/apps/application/flow/step_node/document_split_node/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .impl import * \ No newline at end of file diff --git a/apps/application/flow/step_node/document_split_node/i_document_split_node.py b/apps/application/flow/step_node/document_split_node/i_document_split_node.py deleted file mode 100644 index 7b13d2d405d..00000000000 --- a/apps/application/flow/step_node/document_split_node/i_document_split_node.py +++ /dev/null @@ -1,97 +0,0 @@ -# coding=utf-8 - -from typing import Type - -from django.utils.translation import gettext_lazy as _ -from rest_framework import serializers - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import INode, NodeResult - - -class DocumentSplitNodeSerializer(serializers.Serializer): - document_list = serializers.ListField(required=False, label=_("document list")) - split_strategy = serializers.ChoiceField( - choices=['auto', 'custom', 'qa'], required=False, label=_("split strategy"), default='auto' - ) - paragraph_title_relate_problem_type = serializers.ChoiceField( - choices=['custom', 'referencing'], required=False, label=_("paragraph title relate problem type"), - default='custom' - ) - paragraph_title_relate_problem = serializers.BooleanField( - required=False, label=_("paragraph title relate problem"), default=False - ) - paragraph_title_relate_problem_reference = serializers.ListField( - required=False, label=_("paragraph title relate problem reference"), child=serializers.CharField(), default=[] - ) - document_name_relate_problem_type = serializers.ChoiceField( - choices=['custom', 'referencing'], required=False, label=_("document name relate problem type"), - default='custom' - ) - document_name_relate_problem = serializers.BooleanField( - required=False, label=_("document name relate problem"), default=False - ) - document_name_relate_problem_reference = serializers.ListField( - required=False, label=_("document name relate problem reference"), child=serializers.CharField(), default=[] - ) - limit = serializers.IntegerField(required=False, label=_("limit"), default=4096) - limit_type = serializers.ChoiceField( - choices=['custom', 'referencing'], required=False, label=_("document name relate problem type"), - default='custom' - ) - limit_reference = serializers.ListField( - required=False, label=_("limit reference"), child=serializers.CharField(), default=[] - ) - chunk_size = serializers.IntegerField(required=False, label=_("chunk size"), default=256) - chunk_size_type = serializers.ChoiceField( - choices=['custom', 'referencing'], required=False, label=_("chunk size type"), default='custom' - ) - chunk_size_reference = serializers.ListField( - required=False, label=_("chunk size reference"), child=serializers.CharField(), default=[] - ) - patterns = serializers.ListField( - required=False, label=_("patterns"), child=serializers.CharField(), default=[] - ) - patterns_type = serializers.ChoiceField( - choices=['custom', 'referencing'], required=False, label=_("patterns type"), default='custom' - ) - patterns_reference = serializers.ListField( - required=False, label=_("patterns reference"), child=serializers.CharField(), default=[] - ) - with_filter = serializers.BooleanField( - required=False, label=_("with filter"), default=False - ) - with_filter_type = serializers.ChoiceField( - choices=['custom', 'referencing'], required=False, label=_("with filter type"), default='custom' - ) - with_filter_reference = serializers.ListField( - required=False, label=_("with filter reference"), child=serializers.CharField(), default=[] - ) - - -class IDocumentSplitNode(INode): - type = 'document-split-node' - support = [ - WorkflowMode.APPLICATION, WorkflowMode.APPLICATION_LOOP, WorkflowMode.KNOWLEDGE_LOOP, WorkflowMode.KNOWLEDGE, - WorkflowMode.TOOL, WorkflowMode.TOOL_LOOP - ] - - def get_node_params_serializer_class(self) -> Type[serializers.Serializer]: - return DocumentSplitNodeSerializer - - def _run(self): - if [WorkflowMode.APPLICATION, WorkflowMode.APPLICATION_LOOP, WorkflowMode.TOOL, - WorkflowMode.TOOL_LOOP].__contains__( - self.workflow_manage.flow.workflow_mode): - return self.execute(**self.node_params_serializer.data, **self.flow_params_serializer.data, - **{'knowledge_id': None}) - else: - return self.execute(**self.node_params_serializer.data, **self.flow_params_serializer.data) - - def execute(self, document_list, knowledge_id, split_strategy, paragraph_title_relate_problem_type, - paragraph_title_relate_problem, paragraph_title_relate_problem_reference, - document_name_relate_problem_type, document_name_relate_problem, - document_name_relate_problem_reference, limit, limit_type, limit_reference, chunk_size, chunk_size_type, - chunk_size_reference, patterns, patterns_type, patterns_reference, with_filter, with_filter_type, - with_filter_reference, **kwargs) -> NodeResult: - pass diff --git a/apps/application/flow/step_node/document_split_node/impl/__init__.py b/apps/application/flow/step_node/document_split_node/impl/__init__.py deleted file mode 100644 index cc7dc7dda90..00000000000 --- a/apps/application/flow/step_node/document_split_node/impl/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .base_document_split_node import BaseDocumentSplitNode diff --git a/apps/application/flow/step_node/document_split_node/impl/base_document_split_node.py b/apps/application/flow/step_node/document_split_node/impl/base_document_split_node.py deleted file mode 100644 index 5e71cdd50a1..00000000000 --- a/apps/application/flow/step_node/document_split_node/impl/base_document_split_node.py +++ /dev/null @@ -1,192 +0,0 @@ -# coding=utf-8 -import io -import mimetypes -from typing import List - -from django.core.files.uploadedfile import InMemoryUploadedFile - -from application.flow.i_step_node import NodeResult -from application.flow.step_node.document_split_node.i_document_split_node import IDocumentSplitNode -from common.chunk import text_to_chunk -from knowledge.serializers.document import default_split_handle, FileBufferHandle, md_qa_split_handle - - -def bytes_to_uploaded_file(file_bytes, file_name="file.txt"): - if file_name.startswith("http"): - file_name = "file.txt" - content_type, _ = mimetypes.guess_type(file_name) - if content_type is None: - # 如果未能识别，设置为默认的二进制文件类型 - content_type = "application/octet-stream" - # 创建一个内存中的字节流对象 - file_stream = io.BytesIO(file_bytes) - - # 获取文件大小 - file_size = len(file_bytes) - - # 创建 InMemoryUploadedFile 对象 - uploaded_file = InMemoryUploadedFile( - file=file_stream, - field_name=None, - name=file_name, - content_type=content_type, - size=file_size, - charset=None, - ) - return uploaded_file - - -class BaseDocumentSplitNode(IDocumentSplitNode): - def save_context(self, details, workflow_manage): - self.context['content'] = details.get('content') - self.context['exception_message'] = details.get('err_message') - - def get_reference_content(self, fields: List[str]): - return self.workflow_manage.get_reference_field(fields[0], fields[1:]) if fields else None - - def execute(self, document_list, knowledge_id, split_strategy, paragraph_title_relate_problem_type, - paragraph_title_relate_problem, paragraph_title_relate_problem_reference, - document_name_relate_problem_type, document_name_relate_problem, - document_name_relate_problem_reference, limit, limit_type, limit_reference, chunk_size, chunk_size_type, - chunk_size_reference, patterns, patterns_type, patterns_reference, with_filter, with_filter_type, - with_filter_reference, **kwargs) -> NodeResult: - self.context['knowledge_id'] = knowledge_id - file_list = self.get_reference_content(document_list) - - # 处理引用类型的参数 - if patterns_type == 'referencing': - patterns = self.get_reference_content(patterns_reference) - if limit_type == 'referencing': - limit = self.get_reference_content(limit_reference) - if chunk_size_type == 'referencing': - chunk_size = self.get_reference_content(chunk_size_reference) - if with_filter_type == 'referencing': - with_filter = self.get_reference_content(with_filter_reference) - - paragraph_list = [] - for doc in file_list: - get_buffer = FileBufferHandle().get_buffer - - file_mem = bytes_to_uploaded_file(doc['content'].encode('utf-8'), doc['name']) - if split_strategy == 'qa': - result = md_qa_split_handle.handle(file_mem, get_buffer, self._save_image) - else: - result = default_split_handle.handle(file_mem, patterns, with_filter, limit, get_buffer, - self._save_image) - # 统一处理结果为列表 - results = result if isinstance(result, list) else [result] - - for item in results: - self._process_split_result( - item, knowledge_id, doc.get('id'), doc.get('name'), - split_strategy, paragraph_title_relate_problem_type, - paragraph_title_relate_problem, paragraph_title_relate_problem_reference, - document_name_relate_problem_type, document_name_relate_problem, - document_name_relate_problem_reference, chunk_size - ) - - paragraph_list += results - - self.context['paragraph_list'] = paragraph_list - self.context['document_list'] = file_list - self.context['limit'] = limit - self.context['chunk_size'] = chunk_size - self.context['with_filter'] = with_filter - self.context['patterns'] = patterns - self.context['split_strategy'] = split_strategy - - return NodeResult({'paragraph_list': paragraph_list}, {}) - - def _save_image(self, image_list): - pass - - def _process_split_result( - self, item, knowledge_id, source_file_id, file_name, - split_strategy, paragraph_title_relate_problem_type, - paragraph_title_relate_problem, paragraph_title_relate_problem_reference, - document_name_relate_problem_type, document_name_relate_problem, - document_name_relate_problem_reference, chunk_size - ): - """处理文档分割结果""" - item['meta'] = { - 'knowledge_id': knowledge_id, - 'source_file_id': source_file_id, - 'source_url': file_name, - } - if item.get('name', 'file.txt') == 'file.txt': - item['name'] = file_name - item['source_file_id'] = source_file_id - item['paragraphs'] = item.pop('content', item.get('paragraphs', [])) - - for paragraph in item['paragraphs']: - paragraph['problem_list'] = self._generate_problem_list( - paragraph, file_name, - split_strategy, paragraph_title_relate_problem_type, - paragraph_title_relate_problem, paragraph_title_relate_problem_reference, - document_name_relate_problem_type, document_name_relate_problem, - document_name_relate_problem_reference - ) - paragraph['is_active'] = True - paragraph['chunks'] = text_to_chunk(paragraph['content'], chunk_size) - - def _generate_problem_list( - self, paragraph, document_name, split_strategy, paragraph_title_relate_problem_type, - paragraph_title_relate_problem, paragraph_title_relate_problem_reference, - document_name_relate_problem_type, document_name_relate_problem, - document_name_relate_problem_reference - ): - if paragraph_title_relate_problem_type == 'referencing': - paragraph_title_relate_problem = self.get_reference_content(paragraph_title_relate_problem_reference) - if document_name_relate_problem_type == 'referencing': - document_name_relate_problem = self.get_reference_content(document_name_relate_problem_reference) - - problem_list = [ - item for p in paragraph.get('problem_list', []) for item in p.get('content', '').split('
') - if item.strip() - ] - - if split_strategy == 'auto': - if paragraph_title_relate_problem and paragraph.get('title'): - problem_list.append(paragraph.get('title')) - if document_name_relate_problem and document_name: - problem_list.append(document_name) - elif split_strategy == 'custom': - if paragraph_title_relate_problem and paragraph.get('title'): - problem_list.append(paragraph.get('title')) - if document_name_relate_problem and document_name: - problem_list.append(document_name) - elif split_strategy == 'qa': - if document_name_relate_problem and document_name: - problem_list.append(document_name) - - return list(set(problem_list)) - - def get_details(self, index: int, **kwargs): - paragraph_list = self.context.get('paragraph_list', []) - # 每个文档保留前5个分段 - limited_paragraph_list = [] - for doc in paragraph_list: - if doc.get('paragraphs'): - doc_copy = doc.copy() - doc_copy['paragraphs'] = doc['paragraphs'][:5] - limited_paragraph_list.append(doc_copy) - else: - limited_paragraph_list.append(doc) - paragraph_list = limited_paragraph_list - - return { - 'name': self.node.properties.get('stepName'), - "index": index, - 'run_time': self.context.get('run_time'), - 'type': self.node.type, - 'status': self.status, - 'err_message': self.err_message, - 'paragraph_list': paragraph_list, - 'limit': self.context.get('limit'), - 'chunk_size': self.context.get('chunk_size'), - 'with_filter': self.context.get('with_filter'), - 'patterns': self.context.get('patterns'), - 'split_strategy': self.context.get('split_strategy'), - 'enableException': self.node.properties.get('enableException'), - # 'document_list': self.context.get('document_list', []), - } diff --git a/apps/application/flow/step_node/form_node/__init__.py b/apps/application/flow/step_node/form_node/__init__.py deleted file mode 100644 index ce04b64aea8..00000000000 --- a/apps/application/flow/step_node/form_node/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎 - @file： __init__.py.py - @date：2024/11/4 14:48 - @desc: -""" -from .impl import * \ No newline at end of file diff --git a/apps/application/flow/step_node/form_node/i_form_node.py b/apps/application/flow/step_node/form_node/i_form_node.py deleted file mode 100644 index 9be117f857f..00000000000 --- a/apps/application/flow/step_node/form_node/i_form_node.py +++ /dev/null @@ -1,37 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎 - @file： i_form_node.py - @date：2024/11/4 14:48 - @desc: -""" -from typing import Type - -from rest_framework import serializers - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import INode, NodeResult - -from django.utils.translation import gettext_lazy as _ - - -class FormNodeParamsSerializer(serializers.Serializer): - form_field_list = serializers.ListField(required=True, label=_("Form Configuration")) - form_content_format = serializers.CharField(required=True, label=_('Form output content')) - form_data = serializers.DictField(required=False, allow_null=True, label=_("Form Data")) - - -class IFormNode(INode): - type = 'form-node' - view_type = 'single_view' - support = [WorkflowMode.APPLICATION, WorkflowMode.APPLICATION_LOOP, WorkflowMode.TOOL, WorkflowMode.TOOL_LOOP] - - def get_node_params_serializer_class(self) -> Type[serializers.Serializer]: - return FormNodeParamsSerializer - - def _run(self): - return self.execute(**self.node_params_serializer.data, **self.flow_params_serializer.data) - - def execute(self, form_field_list, form_content_format, form_data, **kwargs) -> NodeResult: - pass diff --git a/apps/application/flow/step_node/form_node/impl/__init__.py b/apps/application/flow/step_node/form_node/impl/__init__.py deleted file mode 100644 index 4cea85e1d9e..00000000000 --- a/apps/application/flow/step_node/form_node/impl/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎 - @file： __init__.py.py - @date：2024/11/4 14:49 - @desc: -""" -from .base_form_node import BaseFormNode diff --git a/apps/application/flow/step_node/form_node/impl/base_form_node.py b/apps/application/flow/step_node/form_node/impl/base_form_node.py deleted file mode 100644 index 710811f1505..00000000000 --- a/apps/application/flow/step_node/form_node/impl/base_form_node.py +++ /dev/null @@ -1,238 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎 - @file： base_form_node.py - @date：2024/11/4 14:52 - @desc: -""" -import copy -import json -import time -from typing import Dict, List - -from langchain_core.prompts import PromptTemplate - -from application.flow.common import Answer -from application.flow.i_step_node import NodeResult -from application.flow.step_node.form_node.i_form_node import IFormNode -import re - -_TEMPLATE_RE = re.compile(r'\{\{([^.\s}]+)\.([^.\s}]+)\}\}') -multi_select_list = [ - 'MultiSelect', - 'MultiRow' -] - - -def get_default_option(option_list, _type, value_field): - try: - if option_list is not None and isinstance(option_list, list) and len(option_list) > 0: - default_value_list = [o.get(value_field) for o in option_list if o.get('default')] - if len(default_value_list) == 0: - return [option_list[0].get( - value_field)] if multi_select_list.__contains__(_type) else option_list[0].get( - value_field) - else: - if multi_select_list.__contains__(_type): - return default_value_list - else: - return default_value_list[0] - except Exception as _: - pass - return [] - - -def write_context(step_variable: Dict, global_variable: Dict, node, workflow): - if step_variable is not None: - for key in step_variable: - node.context[key] = step_variable[key] - if workflow.is_result(node, NodeResult(step_variable, global_variable)) and 'result' in step_variable: - result = step_variable['result'] - yield result - node.answer_text = result - node.context['run_time'] = time.time() - node.context['start_time'] - - -def generate_prompt(workflow_manage, _value): - try: - return workflow_manage.generate_prompt(_value) - except Exception as e: - return _value - - -class BaseFormNode(IFormNode): - def save_context(self, details, workflow_manage): - form_data = details.get('form_data', None) - self.context['result'] = details.get('result') - self.context['form_content_format'] = details.get('form_content_format') - self.context['form_field_list'] = details.get('form_field_list') - self.context['run_time'] = details.get('run_time') - self.context['start_time'] = details.get('start_time') - self.context['form_data'] = form_data - self.context['is_submit'] = details.get('is_submit') - self.context['exception_message'] = details.get('err_message') - if self.node_params.get('is_result', False): - self.answer_text = details.get('result') - if form_data is not None: - for key in form_data: - self.context[key] = form_data[key] - - def reset_field(self, field): - field = copy.copy(field) - reset_field = ['field', 'label', 'default_value'] - for f in reset_field: - _value = field[f] - if _value is None: - continue - if isinstance(_value, str): - field[f] = generate_prompt(self.workflow_manage, _value) - elif f == 'label': - _label_value = _value.get('label') - _value['label'] = generate_prompt(self.workflow_manage, _label_value) - tooltip = _value.get('attrs').get('tooltip') - if tooltip is not None: - _value.get('attrs')['tooltip'] = generate_prompt(self.workflow_manage, tooltip) - - if ['SingleSelect', 'MultiSelect', 'RadioCard', 'RadioRow', 'MultiRow'].__contains__(field.get('input_type')): - if field.get('assignment_method') == 'ref_variables': - option_list = self.workflow_manage.get_reference_field(field.get('option_list')[0], - field.get('option_list')[1:]) - option_list = option_list if isinstance(option_list, list) else [] - field['option_list'] = option_list - field['default_value'] = get_default_option(option_list, field.get('input_type'), - field.get('value_field')) - - if ['JsonInput'].__contains__(field.get('input_type')): - if field.get('default_value_assignment_method') == 'ref_variables': - field['default_value'] = self.workflow_manage.get_reference_field(field.get('default_value')[0], - field.get('default_value')[1:]) - - visibility_rules = field.get('visibility_rules') - if visibility_rules and isinstance(visibility_rules.get('conditions'), list): - for cond in visibility_rules['conditions']: - cond_field = cond.get('field') - if not cond_field or len(cond_field) < 2 or not cond_field[0] or not cond_field[1]: - continue - - # cross node -------> _left - if cond_field[0] != self.node.id: - cond['_left'] = self.workflow_manage.get_reference_field(cond_field[0], cond_field[1:]) - # 右值 {{}} - cond_value = cond.get("value") - if isinstance(cond_value, str) and _TEMPLATE_RE.search(cond_value): - cond['value'] = self._render_cond_value(cond_value) - - return field - - def _render_cond_value(self, value): - """ - render cross-node/global/chat {{}} to literal, preserve same-form {{}} - match.group(0) → "{{开始.question}}" # 完整匹配 - match.group(1) → "开始" # 第一个 () 捕获的 - match.group(2) → "question" # 第二个 () 捕获的 - match.start() → 3 # 匹配起始位置 - match.end() → 16 # 匹配结束位置 - """ - def replacer(match): - node_display = match.group(1) - field_name = match.group(2) - - # field_list: cross_node - for f in self.workflow_manage.field_list: - if f.get('node_name') == node_display and f.get('value') == field_name: - if f.get('node_id') == self.node.id: - return match.group(0) # same node - ref = self.workflow_manage.get_reference_field(f.get('node_id'),[field_name]) - return str(ref) if ref is not None else '' - - # global - if node_display in ('全局变量', 'global'): - for f in self.workflow_manage.global_field_list: - if f.get('value') == field_name: - ref = self.workflow_manage.get_reference_field('global', [field_name]) - return str(ref) if ref is not None else '' - - # chat - if node_display == 'chat': - for f in self.workflow_manage.chat_field_list: - if f.get("value") == field_name: - ref = self.workflow_manage.get_reference_field('chat', [field_name]) - return str(ref) if ref is not None else '' - return match.group(0) - try: - return _TEMPLATE_RE.sub(replacer, value) - except Exception: - return value - - def execute(self, form_field_list, form_content_format, form_data, **kwargs) -> NodeResult: - if form_data is not None: - self.context['is_submit'] = True - self.context['form_data'] = form_data - for key in form_data: - self.context[key] = form_data.get(key) - else: - self.context['is_submit'] = False - form_field_list = [self.reset_field(field) for field in form_field_list] - form_setting = {"form_field_list": form_field_list, "runtime_node_id": self.runtime_node_id, - "chat_record_id": self.flow_params_serializer.data.get("chat_record_id"), - "is_submit": self.context.get("is_submit", False)} - form = f'{json.dumps(form_setting, ensure_ascii=False)}' - context = self.workflow_manage.get_workflow_content() - form_content_format = self.workflow_manage.reset_prompt(form_content_format) - prompt_template = PromptTemplate.from_template(form_content_format, template_format='jinja2') - value = prompt_template.format(form=form, context=context, runtime_node_id=self.runtime_node_id, - chat_record_id=self.flow_params_serializer.data.get("chat_record_id"), - form_field_list=form_field_list) - - return NodeResult( - {'result': value, 'form_field_list': form_field_list, 'form_content_format': form_content_format}, {}, - _write_context=write_context) - - def get_answer_list(self) -> List[Answer] | None: - form_content_format = self.context.get('form_content_format') - form_field_list = self.context.get('form_field_list') - form_setting = {"form_field_list": form_field_list, "runtime_node_id": self.runtime_node_id, - "chat_record_id": self.flow_params_serializer.data.get("chat_record_id"), - 'form_data': self.context.get('form_data', {}), - "is_submit": self.context.get("is_submit", False)} - form = f'{json.dumps(form_setting, ensure_ascii=False)}' - context = self.workflow_manage.get_workflow_content() - form_content_format = self.workflow_manage.reset_prompt(form_content_format) - prompt_template = PromptTemplate.from_template(form_content_format, template_format='jinja2') - value = prompt_template.format(form=form, context=context, runtime_node_id=self.runtime_node_id, - chat_record_id=self.flow_params_serializer.data.get("chat_record_id"), - form_field_list=form_field_list) - return [ - Answer(value, self.view_type, self.runtime_node_id, self.workflow_params.get('chat_record_id') or '', None, - self.runtime_node_id, '')] - - def get_details(self, index: int, **kwargs): - form_content_format = self.context.get('form_content_format') - form_field_list = self.context.get('form_field_list') - form_setting = {"form_field_list": form_field_list, "runtime_node_id": self.runtime_node_id, - "chat_record_id": self.flow_params_serializer.data.get("chat_record_id"), - 'form_data': self.context.get('form_data', {}), - "is_submit": self.context.get("is_submit", False)} - form = f'{json.dumps(form_setting, ensure_ascii=False)}' - context = self.workflow_manage.get_workflow_content() - form_content_format = self.workflow_manage.reset_prompt(form_content_format) - prompt_template = PromptTemplate.from_template(form_content_format, template_format='jinja2') - value = prompt_template.format(form=form, context=context, runtime_node_id=self.runtime_node_id, - chat_record_id=self.flow_params_serializer.data.get("chat_record_id"), - form_field_list=form_field_list) - return { - 'name': self.node.properties.get('stepName'), - "index": index, - "result": value, - "form_content_format": self.context.get('form_content_format'), - "form_field_list": self.context.get('form_field_list'), - 'form_data': self.context.get('form_data'), - 'start_time': self.context.get('start_time'), - 'is_submit': self.context.get('is_submit'), - 'run_time': self.context.get('run_time'), - 'type': self.node.type, - 'status': self.status, - 'err_message': self.err_message, - 'enableException': self.node.properties.get('enableException'), - } diff --git a/apps/application/flow/step_node/image_generate_step_node/__init__.py b/apps/application/flow/step_node/image_generate_step_node/__init__.py deleted file mode 100644 index f3feecc9ce2..00000000000 --- a/apps/application/flow/step_node/image_generate_step_node/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# coding=utf-8 - -from .impl import * diff --git a/apps/application/flow/step_node/image_generate_step_node/i_image_generate_node.py b/apps/application/flow/step_node/image_generate_step_node/i_image_generate_node.py deleted file mode 100644 index 834c842fd14..00000000000 --- a/apps/application/flow/step_node/image_generate_step_node/i_image_generate_node.py +++ /dev/null @@ -1,56 +0,0 @@ -# coding=utf-8 - -from typing import Type - -from django.utils.translation import gettext_lazy as _ -from rest_framework import serializers - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import INode, NodeResult - - -class ImageGenerateNodeSerializer(serializers.Serializer): - model_id = serializers.CharField(required=False, allow_blank=True, allow_null=True, label=_("Model id")) - model_id_type = serializers.CharField(required=False, default='custom', label=_("Model id type")) - model_id_reference = serializers.ListField(required=False, child=serializers.CharField(), allow_empty=True, - label=_("Reference Field")) - prompt = serializers.CharField(required=True, label=_("Prompt word (positive)")) - - negative_prompt = serializers.CharField(required=False, label=_("Prompt word (negative)"), - allow_null=True, allow_blank=True, ) - # 多轮对话数量 - dialogue_number = serializers.IntegerField(required=False, default=0, - label=_("Number of multi-round conversations")) - - dialogue_type = serializers.CharField(required=False, default='NODE', - label=_("Conversation storage type")) - - is_result = serializers.BooleanField(required=False, - label=_('Whether to return content')) - - model_params_setting = serializers.JSONField(required=False, default=dict, - label=_("Model parameter settings")) - - -class IImageGenerateNode(INode): - type = 'image-generate-node' - support = [WorkflowMode.APPLICATION, WorkflowMode.APPLICATION_LOOP, WorkflowMode.KNOWLEDGE, - WorkflowMode.KNOWLEDGE_LOOP, WorkflowMode.TOOL, WorkflowMode.TOOL_LOOP] - - def get_node_params_serializer_class(self) -> Type[serializers.Serializer]: - return ImageGenerateNodeSerializer - - def _run(self): - if [WorkflowMode.KNOWLEDGE, WorkflowMode.KNOWLEDGE_LOOP, WorkflowMode.TOOL, - WorkflowMode.TOOL_LOOP].__contains__( - self.workflow_manage.flow.workflow_mode): - return self.execute(**self.node_params_serializer.data, **self.flow_params_serializer.data, - **{'history_chat_record': [], 'stream': True, 'chat_id': None, 'chat_record_id': None}) - else: - return self.execute(**self.node_params_serializer.data, **self.flow_params_serializer.data) - - def execute(self, model_id, prompt, negative_prompt, dialogue_number, dialogue_type, history_chat_record, - model_params_setting, - chat_record_id, model_id_type=None, model_id_reference=None, - **kwargs) -> NodeResult: - pass diff --git a/apps/application/flow/step_node/image_generate_step_node/impl/__init__.py b/apps/application/flow/step_node/image_generate_step_node/impl/__init__.py deleted file mode 100644 index 14a21a9159c..00000000000 --- a/apps/application/flow/step_node/image_generate_step_node/impl/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# coding=utf-8 - -from .base_image_generate_node import BaseImageGenerateNode diff --git a/apps/application/flow/step_node/image_generate_step_node/impl/base_image_generate_node.py b/apps/application/flow/step_node/image_generate_step_node/impl/base_image_generate_node.py deleted file mode 100644 index bbd0fc1d12a..00000000000 --- a/apps/application/flow/step_node/image_generate_step_node/impl/base_image_generate_node.py +++ /dev/null @@ -1,199 +0,0 @@ -# coding=utf-8 -from functools import reduce -from typing import List - -import requests -from langchain_core.messages import BaseMessage, HumanMessage, AIMessage -from django.utils.translation import gettext_lazy as _ -from application.flow.common import WorkflowMode -from application.flow.i_step_node import NodeResult -from application.flow.step_node.image_generate_step_node.i_image_generate_node import IImageGenerateNode -from common.utils.common import bytes_to_uploaded_file -from knowledge.models import FileSourceType -from models_provider.tools import get_model_instance_by_model_workspace_id -from oss.serializers.file import FileSerializer - - -class BaseImageGenerateNode(IImageGenerateNode): - def save_context(self, details, workflow_manage): - self.context['answer'] = details.get('answer') - self.context['question'] = details.get('question') - self.context['exception_message'] = details.get('err_message') - if self.node_params.get('is_result', False): - self.answer_text = details.get('answer') - - def execute(self, model_id, prompt, negative_prompt, dialogue_number, dialogue_type, history_chat_record, - model_params_setting, - chat_record_id, - model_id_type=None, model_id_reference=None, - **kwargs) -> NodeResult: - # 处理引用类型 - if model_id_type == 'reference' and model_id_reference: - reference_data = self.workflow_manage.get_reference_field( - model_id_reference[0], - model_id_reference[1:], - ) - if reference_data and isinstance(reference_data, dict): - model_id = reference_data.get('model_id', model_id) - model_params_setting = reference_data.get('model_params_setting') - - if model_id is None or model_id == '': - raise Exception(_('Model is not allowed to be empty')) - - workspace_id = self.workflow_manage.get_body().get('workspace_id') - tti_model = get_model_instance_by_model_workspace_id(model_id, workspace_id, - **(model_params_setting or {})) - history_message = self.get_history_message(history_chat_record, dialogue_number) - self.context['history_message'] = history_message - question = self.generate_prompt_question(prompt) - self.context['question'] = question - message_list = self.generate_message_list(question, history_message) - self.context['message_list'] = message_list - self.context['dialogue_type'] = dialogue_type - self.context['negative_prompt'] = self.generate_prompt_question(negative_prompt) - image_urls = tti_model.generate_image(question, negative_prompt) - # 保存图片 - file_urls = [] - for image_url in image_urls: - file_name = 'generated_image.png' - if isinstance(image_url, str): - if image_url.startswith('http'): - # HTTP URL 情况 - image_url = requests.get(image_url).content - elif image_url.startswith('data:image'): - # Data URL 格式 (data:image/png;base64,...) - import base64 - header, encoded = image_url.split(',', 1) - image_url = base64.b64decode(encoded) - else: - import base64 - image_url = base64.b64decode(image_url) - file = bytes_to_uploaded_file(image_url, file_name) - file_url = self.upload_file(file) - file_urls.append(file_url) - self.context['image_list'] = [{'file_id': path.split('/')[-1], 'url': path} for path in file_urls] - answer = ' '.join([f"![Image]({path})" for path in file_urls]) - return NodeResult({'answer': answer, 'chat_model': tti_model, 'message_list': message_list, - 'image': [{'file_id': path.split('/')[-1], 'url': path} for path in file_urls], - 'history_message': history_message, 'question': question}, {}) - - def generate_history_ai_message(self, chat_record): - for val in chat_record.details.values(): - if self.node.id == val['node_id'] and 'image_list' in val: - if val['dialogue_type'] == 'WORKFLOW': - return chat_record.get_ai_message() - image_list = val['image_list'] - return AIMessage(content=[ - *[{'type': 'image_url', 'image_url': {'url': f'{file_url}'}} for file_url in image_list] - ]) - return chat_record.get_ai_message() - - def get_history_message(self, history_chat_record, dialogue_number): - start_index = len(history_chat_record) - dialogue_number - history_message = reduce(lambda x, y: [*x, *y], [ - [self.generate_history_human_message(history_chat_record[index]), - self.generate_history_ai_message(history_chat_record[index])] - for index in - range(start_index if start_index > 0 else 0, len(history_chat_record))], []) - return history_message - - def generate_history_human_message(self, chat_record): - - for data in chat_record.details.values(): - if self.node.id == data['node_id'] and 'image_list' in data: - image_list = data['image_list'] - if len(image_list) == 0 or data['dialogue_type'] == 'WORKFLOW': - return HumanMessage(content=chat_record.problem_text) - return HumanMessage(content=data['question']) - return HumanMessage(content=chat_record.problem_text) - - def generate_prompt_question(self, prompt): - return self.workflow_manage.generate_prompt(prompt) - - def generate_message_list(self, question: str, history_message): - return [ - *history_message, - question - ] - - def upload_file(self, file): - if [WorkflowMode.KNOWLEDGE, WorkflowMode.KNOWLEDGE_LOOP].__contains__( - self.workflow_manage.flow.workflow_mode): - return self.upload_knowledge_file(file) - if [WorkflowMode.TOOL, WorkflowMode.TOOL_LOOP].__contains__(self.workflow_manage.flow.workflow_mode): - return self.upload_tool_file(file) - return self.upload_application_file(file) - - def upload_knowledge_file(self, file): - knowledge_id = self.workflow_params.get('knowledge_id') - meta = { - 'debug': False, - 'knowledge_id': knowledge_id, - } - file_url = FileSerializer(data={ - 'file': file, - 'meta': meta, - 'source_id': knowledge_id, - 'source_type': FileSourceType.KNOWLEDGE.value - }).upload() - return file_url - - def upload_tool_file(self, file): - tool_id = self.workflow_params.get('tool_id') - meta = { - 'debug': False, - 'tool_id': tool_id, - } - file_url = FileSerializer(data={ - 'file': file, - 'meta': meta, - 'source_id': tool_id, - 'source_type': FileSourceType.TOOL.value - }).upload() - return file_url - - def upload_application_file(self, file): - application_id = self.workflow_manage.work_flow_post_handler.chat_info.application_id - chat_id = self.workflow_params.get('chat_id') - meta = { - 'debug': False if application_id else True, - 'chat_id': chat_id, - 'application_id': str(application_id) if application_id else None, - } - file_url = FileSerializer(data={ - 'file': file, - 'meta': meta, - 'source_id': meta['application_id'], - 'source_type': FileSourceType.APPLICATION.value - }).upload() - return file_url - - @staticmethod - def reset_message_list(message_list: List[BaseMessage], answer_text): - result = [{'role': 'user' if isinstance(message, HumanMessage) else 'ai', 'content': message.content} for - message - in - message_list] - result.append({'role': 'ai', 'content': answer_text}) - return result - - def get_details(self, index: int, **kwargs): - return { - 'name': self.node.properties.get('stepName'), - "index": index, - 'run_time': self.context.get('run_time'), - 'history_message': [{'content': message.content, 'role': message.type} for message in - (self.context.get('history_message') if self.context.get( - 'history_message') is not None else [])], - 'question': self.context.get('question'), - 'answer': self.context.get('answer'), - 'type': self.node.type, - 'message_tokens': self.context.get('message_tokens'), - 'answer_tokens': self.context.get('answer_tokens'), - 'status': self.status, - 'err_message': self.err_message, - 'image_list': self.context.get('image_list'), - 'dialogue_type': self.context.get('dialogue_type'), - 'negative_prompt': self.context.get('negative_prompt'), - 'enableException': self.node.properties.get('enableException'), - } diff --git a/apps/application/flow/step_node/image_to_video_step_node/__init__.py b/apps/application/flow/step_node/image_to_video_step_node/__init__.py deleted file mode 100644 index f3feecc9ce2..00000000000 --- a/apps/application/flow/step_node/image_to_video_step_node/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# coding=utf-8 - -from .impl import * diff --git a/apps/application/flow/step_node/image_to_video_step_node/i_image_to_video_node.py b/apps/application/flow/step_node/image_to_video_step_node/i_image_to_video_node.py deleted file mode 100644 index 846f4e90d8f..00000000000 --- a/apps/application/flow/step_node/image_to_video_step_node/i_image_to_video_node.py +++ /dev/null @@ -1,78 +0,0 @@ -# coding=utf-8 - -from typing import Type - -from django.utils.translation import gettext_lazy as _ -from rest_framework import serializers - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import INode, NodeResult - - -class ImageToVideoNodeSerializer(serializers.Serializer): - model_id = serializers.CharField(required=False, allow_blank=True, allow_null=True, label=_("Model id")) - model_id_type = serializers.CharField(required=False, default='custom', label=_("Model id type")) - model_id_reference = serializers.ListField(required=False, child=serializers.CharField(), allow_empty=True, - label=_("Reference Field")) - - prompt = serializers.CharField(required=True, label=_("Prompt word (positive)")) - - negative_prompt = serializers.CharField(required=False, label=_("Prompt word (negative)"), - allow_null=True, allow_blank=True, ) - # 多轮对话数量 - dialogue_number = serializers.IntegerField(required=False, default=0, - label=_("Number of multi-round conversations")) - - dialogue_type = serializers.CharField(required=False, default='NODE', - label=_("Conversation storage type")) - - is_result = serializers.BooleanField(required=False, - label=_('Whether to return content')) - - model_params_setting = serializers.JSONField(required=False, default=dict, - label=_("Model parameter settings")) - - first_frame_url = serializers.ListField(required=True, label=_("First frame url")) - last_frame_url = serializers.ListField(required=False, label=_("Last frame url")) - - -class IImageToVideoNode(INode): - type = 'image-to-video-node' - support = [WorkflowMode.APPLICATION, WorkflowMode.APPLICATION_LOOP, WorkflowMode.KNOWLEDGE, - WorkflowMode.KNOWLEDGE_LOOP, WorkflowMode.TOOL, WorkflowMode.TOOL_LOOP] - - def get_node_params_serializer_class(self) -> Type[serializers.Serializer]: - return ImageToVideoNodeSerializer - - def _run(self): - first_frame_url = self.workflow_manage.get_reference_field( - self.node_params_serializer.data.get('first_frame_url')[0], - self.node_params_serializer.data.get('first_frame_url')[1:]) - if first_frame_url is []: - raise ValueError( - _("First frame url cannot be empty")) - last_frame_url = None - if self.node_params_serializer.data.get('last_frame_url') is not None and self.node_params_serializer.data.get( - 'last_frame_url') != []: - last_frame_url = self.workflow_manage.get_reference_field( - self.node_params_serializer.data.get('last_frame_url')[0], - self.node_params_serializer.data.get('last_frame_url')[1:]) - node_params_data = {k: v for k, v in self.node_params_serializer.data.items() - if k not in ['first_frame_url', 'last_frame_url']} - if [WorkflowMode.KNOWLEDGE, WorkflowMode.KNOWLEDGE_LOOP, WorkflowMode.TOOL, - WorkflowMode.TOOL_LOOP].__contains__( - self.workflow_manage.flow.workflow_mode): - return self.execute(first_frame_url=first_frame_url, last_frame_url=last_frame_url, **node_params_data, - **self.flow_params_serializer.data, - **{'history_chat_record': [], 'stream': True, 'chat_id': None, 'chat_record_id': None}) - else: - return self.execute(first_frame_url=first_frame_url, last_frame_url=last_frame_url, - **node_params_data, **self.flow_params_serializer.data) - - def execute(self, model_id, prompt, negative_prompt, dialogue_number, dialogue_type, history_chat_record, - model_params_setting, - chat_record_id, - first_frame_url, last_frame_url, - model_id_type=None, model_id_reference=None, - **kwargs) -> NodeResult: - pass diff --git a/apps/application/flow/step_node/image_to_video_step_node/impl/__init__.py b/apps/application/flow/step_node/image_to_video_step_node/impl/__init__.py deleted file mode 100644 index 95be14851cb..00000000000 --- a/apps/application/flow/step_node/image_to_video_step_node/impl/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# coding=utf-8 - -from .base_image_to_video_node import BaseImageToVideoNode diff --git a/apps/application/flow/step_node/image_to_video_step_node/impl/base_image_to_video_node.py b/apps/application/flow/step_node/image_to_video_step_node/impl/base_image_to_video_node.py deleted file mode 100644 index 0f19568c6f9..00000000000 --- a/apps/application/flow/step_node/image_to_video_step_node/impl/base_image_to_video_node.py +++ /dev/null @@ -1,213 +0,0 @@ -# coding=utf-8 -import base64 -from functools import reduce -from typing import List - -import requests -from django.db.models import QuerySet -from django.utils.translation import gettext_lazy as _, gettext -from langchain_core.messages import BaseMessage, HumanMessage, AIMessage -from application.flow.common import WorkflowMode -from application.flow.i_step_node import NodeResult -from application.flow.step_node.image_to_video_step_node.i_image_to_video_node import IImageToVideoNode -from common.utils.common import bytes_to_uploaded_file -from knowledge.models import FileSourceType, File -from oss.serializers.file import FileSerializer, mime_types -from models_provider.tools import get_model_instance_by_model_workspace_id - - -class BaseImageToVideoNode(IImageToVideoNode): - def save_context(self, details, workflow_manage): - self.context['answer'] = details.get('answer') - self.context['question'] = details.get('question') - self.context['exception_message'] = details.get('err_message') - if self.node_params.get('is_result', False): - self.answer_text = details.get('answer') - - def execute(self, model_id, prompt, negative_prompt, dialogue_number, dialogue_type, history_chat_record, - model_params_setting, - chat_record_id, - first_frame_url, last_frame_url=None, - model_id_type=None, model_id_reference=None, - **kwargs) -> NodeResult: - # 处理引用类型 - if model_id_type == 'reference' and model_id_reference: - reference_data = self.workflow_manage.get_reference_field( - model_id_reference[0], - model_id_reference[1:], - ) - if reference_data and isinstance(reference_data, dict): - model_id = reference_data.get('model_id', model_id) - model_params_setting = reference_data.get('model_params_setting') - if model_id is None or model_id == '': - raise Exception(_('Model is not allowed to be empty')) - workspace_id = self.workflow_manage.get_body().get('workspace_id') - ttv_model = get_model_instance_by_model_workspace_id(model_id, workspace_id, - **(model_params_setting or {})) - history_message = self.get_history_message(history_chat_record, dialogue_number) - self.context['history_message'] = history_message - question = self.generate_prompt_question(prompt) - self.context['question'] = question - message_list = self.generate_message_list(question, history_message) - self.context['message_list'] = message_list - self.context['dialogue_type'] = dialogue_type - self.context['negative_prompt'] = self.generate_prompt_question(negative_prompt) - self.context['first_frame_url'] = first_frame_url - self.context['last_frame_url'] = last_frame_url - # 处理首尾帧图片 这块可以是url 也可以是file_id 如果是url 可以直接传递给模型 如果是file_id 需要传base64 - # 判断是不是 url - first_frame_url = self.get_file_base64(first_frame_url) - last_frame_url = self.get_file_base64(last_frame_url) - video_urls = ttv_model.generate_video(question, negative_prompt, first_frame_url, last_frame_url) - # 保存图片 - if video_urls is None or video_urls == '': - return NodeResult({'answer': gettext('Failed to generate video')}, {}) - file_name = 'generated_video.mp4' - if isinstance(video_urls, str) and video_urls.startswith('http'): - video_urls = requests.get(video_urls).content - file = bytes_to_uploaded_file(video_urls, file_name) - file_url = self.upload_file(file) - video_label = f'' - video_list = [{'file_id': file_url.split('/')[-1], 'file_name': file_name, 'url': file_url}] - return NodeResult({'answer': video_label, 'chat_model': ttv_model, 'message_list': message_list, - 'video': video_list, - 'history_message': history_message, 'question': question}, {}) - - def get_file_base64(self, image_url): - try: - if isinstance(image_url, list): - image_url = image_url[0].get('file_id') if 'file_id' in image_url[0] else image_url[0].get('url') - if isinstance(image_url, str) and not image_url.startswith('http'): - file = QuerySet(File).filter(id=image_url).first() - file_bytes = file.get_bytes() - # 如果我不知道content_type 可以用 magic 库去检测 - file_type = file.file_name.split(".")[-1].lower() - content_type = mime_types.get(file_type, 'application/octet-stream') - encoded_bytes = base64.b64encode(file_bytes) - return f'data:{content_type};base64,{encoded_bytes.decode()}' - return image_url - except Exception as e: - raise ValueError( - gettext("Failed to obtain the image")) - - def upload_file(self, file): - if [WorkflowMode.KNOWLEDGE, WorkflowMode.KNOWLEDGE_LOOP].__contains__( - self.workflow_manage.flow.workflow_mode): - return self.upload_knowledge_file(file) - if [WorkflowMode.TOOL, WorkflowMode.TOOL_LOOP].__contains__(self.workflow_manage.flow.workflow_mode): - return self.upload_tool_file(file) - return self.upload_application_file(file) - - def upload_knowledge_file(self, file): - knowledge_id = self.workflow_params.get('knowledge_id') - meta = { - 'debug': False, - 'knowledge_id': knowledge_id - } - file_url = FileSerializer(data={ - 'file': file, - 'meta': meta, - 'source_id': knowledge_id, - 'source_type': FileSourceType.KNOWLEDGE.value - }).upload() - return file_url - - def upload_tool_file(self, file): - tool_id = self.workflow_params.get('tool_id') - meta = { - 'debug': False, - 'tool_id': tool_id, - } - file_url = FileSerializer(data={ - 'file': file, - 'meta': meta, - 'source_id': tool_id, - 'source_type': FileSourceType.TOOL.value - }).upload() - return file_url - - def upload_application_file(self, file): - application_id = self.workflow_manage.work_flow_post_handler.chat_info.application_id - chat_id = self.workflow_params.get('chat_id') - meta = { - 'debug': False if application_id else True, - 'chat_id': chat_id, - 'application_id': str(application_id) if application_id else None, - } - file_url = FileSerializer(data={ - 'file': file, - 'meta': meta, - 'source_id': meta['application_id'], - 'source_type': FileSourceType.APPLICATION.value - }).upload() - return file_url - - def generate_history_ai_message(self, chat_record): - for val in chat_record.details.values(): - if self.node.id == val['node_id'] and 'image_list' in val: - if val['dialogue_type'] == 'WORKFLOW': - return chat_record.get_ai_message() - image_list = val['image_list'] - return AIMessage(content=[ - *[{'type': 'image_url', 'image_url': {'url': f'{file_url}'}} for file_url in image_list] - ]) - return chat_record.get_ai_message() - - def get_history_message(self, history_chat_record, dialogue_number): - start_index = len(history_chat_record) - dialogue_number - history_message = reduce(lambda x, y: [*x, *y], [ - [self.generate_history_human_message(history_chat_record[index]), - self.generate_history_ai_message(history_chat_record[index])] - for index in - range(start_index if start_index > 0 else 0, len(history_chat_record))], []) - return history_message - - def generate_history_human_message(self, chat_record): - - for data in chat_record.details.values(): - if self.node.id == data['node_id'] and 'image_list' in data: - image_list = data['image_list'] - if len(image_list) == 0 or data['dialogue_type'] == 'WORKFLOW': - return HumanMessage(content=chat_record.problem_text) - return HumanMessage(content=data['question']) - return HumanMessage(content=chat_record.problem_text) - - def generate_prompt_question(self, prompt): - return self.workflow_manage.generate_prompt(prompt) - - def generate_message_list(self, question: str, history_message): - return [ - *history_message, - question - ] - - @staticmethod - def reset_message_list(message_list: List[BaseMessage], answer_text): - result = [{'role': 'user' if isinstance(message, HumanMessage) else 'ai', 'content': message.content} for - message - in - message_list] - result.append({'role': 'ai', 'content': answer_text}) - return result - - def get_details(self, index: int, **kwargs): - return { - 'name': self.node.properties.get('stepName'), - "index": index, - 'run_time': self.context.get('run_time'), - 'history_message': [{'content': message.content, 'role': message.type} for message in - (self.context.get('history_message') if self.context.get( - 'history_message') is not None else [])], - 'question': self.context.get('question'), - 'answer': self.context.get('answer'), - 'type': self.node.type, - 'message_tokens': self.context.get('message_tokens'), - 'answer_tokens': self.context.get('answer_tokens'), - 'status': self.status, - 'err_message': self.err_message, - 'first_frame_url': self.context.get('first_frame_url'), - 'last_frame_url': self.context.get('last_frame_url'), - 'dialogue_type': self.context.get('dialogue_type'), - 'negative_prompt': self.context.get('negative_prompt'), - 'enableException': self.node.properties.get('enableException'), - } diff --git a/apps/application/flow/step_node/image_understand_step_node/__init__.py b/apps/application/flow/step_node/image_understand_step_node/__init__.py deleted file mode 100644 index f3feecc9ce2..00000000000 --- a/apps/application/flow/step_node/image_understand_step_node/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# coding=utf-8 - -from .impl import * diff --git a/apps/application/flow/step_node/image_understand_step_node/i_image_understand_node.py b/apps/application/flow/step_node/image_understand_step_node/i_image_understand_node.py deleted file mode 100644 index 907ad019a33..00000000000 --- a/apps/application/flow/step_node/image_understand_step_node/i_image_understand_node.py +++ /dev/null @@ -1,63 +0,0 @@ -# coding=utf-8 - -from typing import Type - -from rest_framework import serializers - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import INode, NodeResult - -from django.utils.translation import gettext_lazy as _ - - -class ImageUnderstandNodeSerializer(serializers.Serializer): - model_id = serializers.CharField(required=False, allow_blank=True, allow_null=True, label=_("Model id")) - model_id_type = serializers.CharField(required=False, default='custom', label=_("Model id type")) - model_id_reference = serializers.ListField(required=False, child=serializers.CharField(), allow_empty=True, - label=_("Reference Field")) - system = serializers.CharField(required=False, allow_blank=True, allow_null=True, - label=_("Role Setting")) - prompt = serializers.CharField(required=True, label=_("Prompt word")) - # 多轮对话数量 - dialogue_number = serializers.IntegerField(required=True, label=_("Number of multi-round conversations")) - - dialogue_type = serializers.CharField(required=True, label=_("Conversation storage type")) - - is_result = serializers.BooleanField(required=False, - label=_('Whether to return content')) - - image_list = serializers.ListField(required=False, label=_("picture")) - - model_params_setting = serializers.JSONField(required=False, default=dict, - label=_("Model parameter settings")) - model_setting = serializers.DictField(required=False, - label='Model settings') - - -class IImageUnderstandNode(INode): - type = 'image-understand-node' - support = [WorkflowMode.APPLICATION, WorkflowMode.APPLICATION_LOOP, WorkflowMode.KNOWLEDGE, - WorkflowMode.KNOWLEDGE_LOOP, WorkflowMode.TOOL, WorkflowMode.TOOL_LOOP] - - def get_node_params_serializer_class(self) -> Type[serializers.Serializer]: - return ImageUnderstandNodeSerializer - - def _run(self): - res = self.workflow_manage.get_reference_field(self.node_params_serializer.data.get('image_list')[0], - self.node_params_serializer.data.get('image_list')[1:]) - if [WorkflowMode.KNOWLEDGE, WorkflowMode.KNOWLEDGE_LOOP, WorkflowMode.TOOL, - WorkflowMode.TOOL_LOOP].__contains__( - self.workflow_manage.flow.workflow_mode): - return self.execute(image=res, **self.node_params_serializer.data, **self.flow_params_serializer.data, - **{'history_chat_record': [], 'stream': True, 'chat_record_id': None}) - else: - return self.execute(image=res, **self.node_params_serializer.data, **self.flow_params_serializer.data) - - def execute(self, model_id, system, prompt, dialogue_number, dialogue_type, history_chat_record, stream, - model_params_setting, - chat_record_id, - image, - model_id_type=None, model_id_reference=None, - model_setting=None, - **kwargs) -> NodeResult: - pass diff --git a/apps/application/flow/step_node/image_understand_step_node/impl/__init__.py b/apps/application/flow/step_node/image_understand_step_node/impl/__init__.py deleted file mode 100644 index ba251283921..00000000000 --- a/apps/application/flow/step_node/image_understand_step_node/impl/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# coding=utf-8 - -from .base_image_understand_node import BaseImageUnderstandNode diff --git a/apps/application/flow/step_node/image_understand_step_node/impl/base_image_understand_node.py b/apps/application/flow/step_node/image_understand_step_node/impl/base_image_understand_node.py deleted file mode 100644 index 00ed9b3d128..00000000000 --- a/apps/application/flow/step_node/image_understand_step_node/impl/base_image_understand_node.py +++ /dev/null @@ -1,340 +0,0 @@ -# coding=utf-8 -import base64 -import time -from functools import reduce -from typing import List, Dict - -from django.db.models import QuerySet -from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage, AIMessage -from django.utils.translation import gettext_lazy as _ -from application.flow.i_step_node import NodeResult, INode -from application.flow.step_node.image_understand_step_node.i_image_understand_node import IImageUnderstandNode -from application.flow.tools import Reasoning -from common.utils.common import guess_image_format -from knowledge.models import File -from models_provider.tools import get_model_instance_by_model_workspace_id - - -def _write_context(node_variable: Dict, workflow_variable: Dict, node: INode, workflow, answer: str, - reasoning_content: str): - chat_model = node_variable.get('chat_model') - message_tokens = chat_model.get_num_tokens_from_messages(node_variable.get('message_list')) - answer_tokens = chat_model.get_num_tokens(answer) - node.context['message_tokens'] = message_tokens - node.context['answer_tokens'] = answer_tokens - node.context['answer'] = answer - node.context['history_message'] = node_variable['history_message'] - node.context['question'] = node_variable['question'] - node.context['run_time'] = time.time() - node.context['start_time'] - node.context['reasoning_content'] = reasoning_content - if workflow.is_result(node, NodeResult(node_variable, workflow_variable)): - node.answer_text = answer - - -def write_context_stream(node_variable: Dict, workflow_variable: Dict, node: INode, workflow): - """ - 写入上下文数据 (流式) - @param node_variable: 节点数据 - @param workflow_variable: 全局数据 - @param node: 节点 - @param workflow: 工作流管理器 - """ - response = node_variable.get('result') - answer = '' - reasoning_content = '' - model_setting = node.context.get('model_setting', - {'reasoning_content_enable': False, 'reasoning_content_end': '', - 'reasoning_content_start': ''}) - reasoning = Reasoning(model_setting.get('reasoning_content_start', ''), - model_setting.get('reasoning_content_end', '')) - response_reasoning_content = False - - for chunk in response: - if workflow.is_the_task_interrupted(): - break - - # 处理 reasoning content - reasoning_chunk = reasoning.get_reasoning_content(chunk) - content_chunk = reasoning_chunk.get('content') - if 'reasoning_content' in chunk.additional_kwargs: - response_reasoning_content = True - reasoning_content_chunk = chunk.additional_kwargs.get('reasoning_content', '') - else: - reasoning_content_chunk = reasoning_chunk.get('reasoning_content') - - answer += content_chunk - if reasoning_content_chunk is None: - reasoning_content_chunk = '' - reasoning_content += reasoning_content_chunk - - # 处理 chunk.content 为 list 的情况 - if isinstance(chunk.content, list): - for chunk_item in chunk.content: - text = chunk_item.get("text", "") - yield {'content': text, - 'reasoning_content': reasoning_content_chunk if model_setting.get('reasoning_content_enable', - False) else ''} - else: - text = chunk.content or "" - yield {'content': text, - 'reasoning_content': reasoning_content_chunk if model_setting.get('reasoning_content_enable', - False) else ''} - - reasoning_chunk = reasoning.get_end_reasoning_content() - answer += reasoning_chunk.get('content') - reasoning_content_chunk = "" - if not response_reasoning_content: - reasoning_content_chunk = reasoning_chunk.get( - 'reasoning_content') - yield {'content': reasoning_chunk.get('content'), - 'reasoning_content': reasoning_content_chunk if model_setting.get('reasoning_content_enable', - False) else ''} - _write_context(node_variable, workflow_variable, node, workflow, answer, reasoning_content) - - -def write_context(node_variable: Dict, workflow_variable: Dict, node: INode, workflow): - """ - 写入上下文数据 - @param node_variable: 节点数据 - @param workflow_variable: 全局数据 - @param node: 节点实例对象 - @param workflow: 工作流管理器 - """ - response = node_variable.get('result') - model_setting = node.context.get('model_setting', - {'reasoning_content_enable': False, 'reasoning_content_end': '', - 'reasoning_content_start': ''}) - reasoning = Reasoning(model_setting.get('reasoning_content_start'), model_setting.get('reasoning_content_end')) - reasoning_result = reasoning.get_reasoning_content(response) - reasoning_result_end = reasoning.get_end_reasoning_content() - content = reasoning_result.get('content') + reasoning_result_end.get('content') - meta = {**response.response_metadata, **response.additional_kwargs} - if 'reasoning_content' in meta: - reasoning_content = (meta.get('reasoning_content', '') or '') - else: - reasoning_content = (reasoning_result.get('reasoning_content') or '') + ( - reasoning_result_end.get('reasoning_content') or '') - _write_context(node_variable, workflow_variable, node, workflow, content, reasoning_content) - - -def file_id_to_base64(file_id: str): - file = QuerySet(File).filter(id=file_id).first() - file_bytes = file.get_bytes() - base64_image = base64.b64encode(file_bytes).decode("utf-8") - return [base64_image, guess_image_format(file_bytes, file.file_name)] - - -class BaseImageUnderstandNode(IImageUnderstandNode): - def save_context(self, details, workflow_manage): - self.context['answer'] = details.get('answer') - self.context['question'] = details.get('question') - self.context['exception_message'] = details.get('err_message') - if self.node_params.get('is_result', False): - self.answer_text = details.get('answer') - - def execute(self, model_id, system, prompt, dialogue_number, dialogue_type, history_chat_record, stream, - model_params_setting, - chat_record_id, - image, - model_id_type=None, model_id_reference=None, - model_setting=None, - **kwargs) -> NodeResult: - if model_setting is None: - model_setting = {'reasoning_content_enable': False, 'reasoning_content_end': '', - 'reasoning_content_start': ''} - self.context['model_setting'] = model_setting - # 处理引用类型 - if model_id_type == 'reference' and model_id_reference: - reference_data = self.workflow_manage.get_reference_field( - model_id_reference[0], - model_id_reference[1:], - ) - if reference_data and isinstance(reference_data, dict): - model_id = reference_data.get('model_id', model_id) - model_params_setting = reference_data.get('model_params_setting') - - if model_id is None or model_id == '': - raise Exception(_('Model is not allowed to be empty')) - # 处理不正确的参数 - workspace_id = self.workflow_manage.get_body().get('workspace_id') - image_model = get_model_instance_by_model_workspace_id(model_id, workspace_id, - **(model_params_setting or {})) - # 执行详情中的历史消息不需要图片内容 - history_message = self.get_history_message_for_details(history_chat_record, dialogue_number) - self.context['history_message'] = history_message - question = self.generate_prompt_question(prompt) - self.context['question'] = question.content - system = self.workflow_manage.generate_prompt(system) - self.context['system'] = system - # 生成消息列表, 真实的history_message - message_list = self.generate_message_list(image_model, system, prompt, - self.get_history_message(history_chat_record, dialogue_number), image) - self.context['message_list'] = message_list - self.generate_context_image(image) - self.context['dialogue_type'] = dialogue_type - if stream: - r = image_model.stream(message_list) - return NodeResult({'result': r, 'chat_model': image_model, 'message_list': message_list, - 'history_message': history_message, 'question': question.content}, {}, - _write_context=write_context_stream) - else: - r = image_model.invoke(message_list) - return NodeResult({'result': r, 'chat_model': image_model, 'message_list': message_list, - 'history_message': history_message, 'question': question.content}, {}, - _write_context=write_context) - - def generate_context_image(self, image): - if isinstance(image, str) and image.startswith('http'): - self.context['image_list'] = [{'url': image}] - elif image is not None and len(image) > 0: - self.context['image_list'] = image - - def get_history_message_for_details(self, history_chat_record, dialogue_number): - start_index = len(history_chat_record) - dialogue_number - history_message = reduce(lambda x, y: [*x, *y], [ - [self.generate_history_human_message_for_details(history_chat_record[index]), - self.generate_history_ai_message(history_chat_record[index])] - for index in - range(start_index if start_index > 0 else 0, len(history_chat_record))], []) - return history_message - - def generate_history_ai_message(self, chat_record): - for val in chat_record.details.values(): - if self.node.id == val['node_id'] and 'image_list' in val: - if val['dialogue_type'] == 'WORKFLOW': - return chat_record.get_ai_message() - return AIMessage(content=val['answer']) - return chat_record.get_ai_message() - - def generate_history_human_message_for_details(self, chat_record): - for data in chat_record.details.values(): - if self.node.id == data['node_id'] and 'image_list' in data: - image_list = data['image_list'] or [] - if len(image_list) == 0 or data['dialogue_type'] == 'WORKFLOW': - return HumanMessage(content=chat_record.problem_text) - - file_id_list = [] - url_list = [] - for image in image_list: - if 'file_id' in image: - file_id_list.append(image.get('file_id')) - elif 'url' in image: - url_list.append(image.get('url')) - return HumanMessage(content=[ - {'type': 'text', 'text': data['question']}, - *[{'type': 'image_url', 'image_url': {'url': f'./oss/file/{file_id}'}} for file_id in file_id_list], - *[{'type': 'image_url', 'image_url': {'url': url}} for url in url_list] - ]) - return HumanMessage(content=chat_record.problem_text) - - def get_history_message(self, history_chat_record, dialogue_number): - start_index = len(history_chat_record) - dialogue_number - history_message = reduce(lambda x, y: [*x, *y], [ - [self.generate_history_human_message(history_chat_record[index]), - self.generate_history_ai_message(history_chat_record[index])] - for index in - range(start_index if start_index > 0 else 0, len(history_chat_record))], []) - return history_message - - def generate_history_human_message(self, chat_record): - - for data in chat_record.details.values(): - if self.node.id == data['node_id'] and 'image_list' in data: - image_list = data['image_list'] or [] - if len(image_list) == 0 or data['dialogue_type'] == 'WORKFLOW': - return HumanMessage(content=chat_record.problem_text) - file_id_list = [] - url_list = [] - for image in image_list: - if 'file_id' in image: - file_id_list.append(image.get('file_id')) - elif 'url' in image: - url_list.append(image.get('url')) - image_base64_list = [file_id_to_base64(file_id) for file_id in file_id_list] - - return HumanMessage( - content=[ - {'type': 'text', 'text': data['question']}, - *[{'type': 'image_url', - 'image_url': {'url': f'data:image/{base64_image[1]};base64,{base64_image[0]}'}} for - base64_image in image_base64_list], - *[{'type': 'image_url', 'image_url': {'url': url}} for url in url_list] - ]) - return HumanMessage(content=chat_record.problem_text) - - def generate_prompt_question(self, prompt): - return HumanMessage(self.workflow_manage.generate_prompt(prompt)) - - def _process_images(self, image): - """ - 处理图像数据，转换为模型可识别的格式 - """ - images = [] - if isinstance(image, str) and image.startswith('http'): - images.append({'type': 'image_url', 'image_url': {'url': image}}) - elif image is not None and len(image) > 0: - for img in image: - if 'file_id' in img: - file_id = img['file_id'] - file = QuerySet(File).filter(id=file_id).first() - image_bytes = file.get_bytes() - base64_image = base64.b64encode(image_bytes).decode("utf-8") - image_format = guess_image_format(image_bytes, file.file_name) - images.append( - {'type': 'image_url', 'image_url': {'url': f'data:image/{image_format};base64,{base64_image}'}}) - elif 'url' in img and img['url'].startswith('http'): - images.append( - {'type': 'image_url', 'image_url': {'url': img["url"]}}) - return images - - def generate_message_list(self, image_model, system: str, prompt: str, history_message, image): - prompt_text = self.workflow_manage.generate_prompt(prompt) - images = self._process_images(image) - - if images: - messages = [HumanMessage(content=[{'type': 'text', 'text': prompt_text}, *images])] - else: - messages = [HumanMessage(prompt_text)] - - if system is not None and len(system) > 0: - return [ - SystemMessage(system), - *history_message, - *messages - ] - else: - return [ - *history_message, - *messages - ] - - @staticmethod - def reset_message_list(message_list: List[BaseMessage], answer_text): - result = [{'role': 'user' if isinstance(message, HumanMessage) else 'ai', 'content': message.content} for - message - in - message_list] - result.append({'role': 'ai', 'content': answer_text}) - return result - - def get_details(self, index: int, **kwargs): - return { - 'name': self.node.properties.get('stepName'), - "index": index, - 'run_time': self.context.get('run_time'), - 'system': self.context.get('system'), - 'history_message': [{'content': message.content, 'role': message.type} for message in - (self.context.get('history_message') if self.context.get( - 'history_message') is not None else [])], - 'question': self.context.get('question'), - 'answer': self.context.get('answer'), - 'reasoning_content': self.context.get('reasoning_content'), - 'type': self.node.type, - 'message_tokens': self.context.get('message_tokens'), - 'answer_tokens': self.context.get('answer_tokens'), - 'status': self.status, - 'err_message': self.err_message, - 'image_list': self.context.get('image_list'), - 'dialogue_type': self.context.get('dialogue_type'), - 'enableException': self.node.properties.get('enableException'), - } diff --git a/apps/application/flow/step_node/intent_node/__init__.py b/apps/application/flow/step_node/intent_node/__init__.py deleted file mode 100644 index 4b372238e7d..00000000000 --- a/apps/application/flow/step_node/intent_node/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# coding=utf-8 - - - - -from .impl import * \ No newline at end of file diff --git a/apps/application/flow/step_node/intent_node/i_intent_node.py b/apps/application/flow/step_node/intent_node/i_intent_node.py deleted file mode 100644 index d22d321c842..00000000000 --- a/apps/application/flow/step_node/intent_node/i_intent_node.py +++ /dev/null @@ -1,59 +0,0 @@ -# coding=utf-8 - -from typing import Type - -from django.utils.translation import gettext_lazy as _ -from rest_framework import serializers - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import INode, NodeResult - - -class IntentBranchSerializer(serializers.Serializer): - id = serializers.CharField(required=True, label=_("Branch id")) - content = serializers.CharField(required=True, label=_("content")) - isOther = serializers.BooleanField(required=True, label=_("Branch Type")) - - -class IntentNodeSerializer(serializers.Serializer): - model_id = serializers.CharField(required=False, allow_blank=True, allow_null=True, label=_("Model id")) - model_id_type = serializers.CharField(required=False, default='custom', label=_("Model id type")) - model_id_reference = serializers.ListField(required=False, child=serializers.CharField(), allow_empty=True, - label=_("Reference Field")) - content_list = serializers.ListField(required=True, label=_("Text content")) - dialogue_number = serializers.IntegerField(required=True, label= - _("Number of multi-round conversations")) - model_params_setting = serializers.DictField(required=False, - label=_("Model parameter settings")) - branch = IntentBranchSerializer(many=True) - - -class IIntentNode(INode): - type = 'intent-node' - support = [WorkflowMode.APPLICATION, WorkflowMode.APPLICATION_LOOP, WorkflowMode.KNOWLEDGE, - WorkflowMode.KNOWLEDGE_LOOP, WorkflowMode.TOOL, WorkflowMode.TOOL_LOOP] - - def save_context(self, details, workflow_manage): - pass - - def get_node_params_serializer_class(self) -> Type[serializers.Serializer]: - return IntentNodeSerializer - - def _run(self): - question = self.workflow_manage.get_reference_field( - self.node_params_serializer.data.get('content_list')[0], - self.node_params_serializer.data.get('content_list')[1:], - ) - if [WorkflowMode.KNOWLEDGE, WorkflowMode.KNOWLEDGE_LOOP, WorkflowMode.TOOL, - WorkflowMode.TOOL_LOOP].__contains__( - self.workflow_manage.flow.workflow_mode): - return self.execute(**self.node_params_serializer.data, **self.flow_params_serializer.data, - **{'history_chat_record': [], 'stream': True, 'chat_id': None, 'chat_record_id': None, - 'user_input': str(question)}) - else: - return self.execute(**self.node_params_serializer.data, **self.flow_params_serializer.data, - user_input=str(question)) - - def execute(self, model_id, dialogue_number, history_chat_record, user_input, branch, - model_params_setting=None, model_id_type=None, model_id_reference=None, **kwargs) -> NodeResult: - pass diff --git a/apps/application/flow/step_node/intent_node/impl/__init__.py b/apps/application/flow/step_node/intent_node/impl/__init__.py deleted file mode 100644 index 56954da75d4..00000000000 --- a/apps/application/flow/step_node/intent_node/impl/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ - - -from .base_intent_node import BaseIntentNode \ No newline at end of file diff --git a/apps/application/flow/step_node/intent_node/impl/base_intent_node.py b/apps/application/flow/step_node/intent_node/impl/base_intent_node.py deleted file mode 100644 index 872e180efdd..00000000000 --- a/apps/application/flow/step_node/intent_node/impl/base_intent_node.py +++ /dev/null @@ -1,266 +0,0 @@ -# coding=utf-8 -import json -import re -import time -from typing import List, Dict, Any -from functools import reduce - -from django.db.models import QuerySet -from django.utils.translation import gettext_lazy as _ -from langchain_core.messages import HumanMessage, SystemMessage - -from application.flow.i_step_node import INode, NodeResult -from application.flow.step_node.intent_node.i_intent_node import IIntentNode -from models_provider.models import Model -from models_provider.tools import get_model_instance_by_model_workspace_id, get_model_credential -from .prompt_template import PROMPT_TEMPLATE - - -def get_default_model_params_setting(model_id): - model = QuerySet(Model).filter(id=model_id).first() - credential = get_model_credential(model.provider, model.model_type, model.model_name) - model_params_setting = credential.get_model_params_setting_form( - model.model_name).get_default_form_data() - return model_params_setting - - -def _write_context(node_variable: Dict, workflow_variable: Dict, node: INode, workflow, answer: str): - chat_model = node_variable.get('chat_model') - message_tokens = chat_model.get_num_tokens_from_messages(node_variable.get('message_list')) - answer_tokens = chat_model.get_num_tokens(answer) - - node.context['message_tokens'] = message_tokens - node.context['answer_tokens'] = answer_tokens - node.context['answer'] = answer - node.context['history_message'] = node_variable['history_message'] - node.context['user_input'] = node_variable['user_input'] - node.context['branch_id'] = node_variable.get('branch_id') - node.context['reason'] = node_variable.get('reason') - node.context['category'] = node_variable.get('category') - node.context['run_time'] = time.time() - node.context['start_time'] - - -def write_context(node_variable: Dict, workflow_variable: Dict, node: INode, workflow): - response = node_variable.get('result') - answer = response.content - _write_context(node_variable, workflow_variable, node, workflow, answer) - - -class BaseIntentNode(IIntentNode): - - def save_context(self, details, workflow_manage): - self.context['exception_message'] = details.get('err_message') - self.context['branch_id'] = details.get('branch_id') - self.context['category'] = details.get('category') - - def execute(self, model_id, dialogue_number, history_chat_record, user_input, branch, - model_params_setting=None, model_id_type=None, model_id_reference=None, **kwargs) -> NodeResult: - # 处理引用类型 - if model_id_type == 'reference' and model_id_reference: - reference_data = self.workflow_manage.get_reference_field( - model_id_reference[0], - model_id_reference[1:], - ) - if reference_data and isinstance(reference_data, dict): - model_id = reference_data.get('model_id', model_id) - model_params_setting = reference_data.get('model_params_setting') - if not model_id: - raise Exception(_('Model is not allowed to be empty')) - - # 设置默认模型参数 - if model_params_setting is None and model_id: - model_params_setting = get_default_model_params_setting(model_id) - - # 获取模型实例 - workspace_id = self.workflow_manage.get_body().get('workspace_id') - chat_model = get_model_instance_by_model_workspace_id( - model_id, workspace_id, **(model_params_setting or {}) - ) - - # 获取历史对话 - history_message = self.get_history_message(history_chat_record, dialogue_number) - self.context['history_message'] = history_message - - # 保存问题到上下文 - self.context['user_input'] = user_input - - # 构建分类提示词 - prompt = self.build_classification_prompt(user_input, branch) - - # 生成消息列表 - system = self.build_system_prompt() - message_list = self.generate_message_list(system, prompt, history_message) - self.context['message_list'] = message_list - - # 调用模型进行分类 - try: - r = chat_model.invoke(message_list) - classification_result = r.content.strip() - # 解析分类结果获取分支信息 - matched_branch = self.parse_classification_result(classification_result, branch) - - # 返回结果 - return NodeResult({ - 'result': r, - 'chat_model': chat_model, - 'message_list': message_list, - 'history_message': history_message, - 'user_input': user_input, - 'branch_id': matched_branch['id'], - 'reason': self.parse_result_reason(r.content), - 'category': matched_branch.get('content', matched_branch['id']) - }, {}, _write_context=write_context) - - except Exception as e: - # 错误处理：返回"其他"分支 - other_branch = self.find_other_branch(branch) - if other_branch: - return NodeResult({ - 'branch_id': other_branch['id'], - 'category': other_branch.get('content', other_branch['id']), - 'error': str(e) - }, {}) - else: - raise Exception(f"error: {str(e)}") - - @staticmethod - def get_history_message(history_chat_record, dialogue_number): - """获取历史消息""" - start_index = len(history_chat_record) - dialogue_number - history_message = reduce(lambda x, y: [*x, *y], [ - [history_chat_record[index].get_human_message(), history_chat_record[index].get_ai_message()] - for index in - range(start_index if start_index > 0 else 0, len(history_chat_record))], []) - - for message in history_message: - if isinstance(message.content, str): - message.content = re.sub(r'.*?', '', message.content, flags=re.DOTALL) - return history_message - - def build_system_prompt(self) -> str: - """构建系统提示词""" - return "你是一个专业的意图识别助手，请根据用户输入和意图选项，准确识别用户的真实意图。" - - def build_classification_prompt(self, user_input: str, branch: List[Dict]) -> str: - """构建分类提示词""" - - classification_list = [] - - other_branch = self.find_other_branch(branch) - # 添加其他分支 - if other_branch: - classification_list.append({ - "classificationId": 0, - "content": other_branch.get('content') - }) - # 添加正常分支 - classification_id = 1 - for b in branch: - if not b.get('isOther'): - classification_list.append({ - "classificationId": classification_id, - "content": b['content'] - }) - classification_id += 1 - - return PROMPT_TEMPLATE.format( - classification_list=classification_list, - user_input=user_input - ) - - def generate_message_list(self, system: str, prompt: str, history_message): - """生成消息列表""" - if system is None or len(system) == 0: - return [*history_message, HumanMessage(self.workflow_manage.generate_prompt(prompt))] - else: - return [SystemMessage(self.workflow_manage.generate_prompt(system)), *history_message, - HumanMessage(self.workflow_manage.generate_prompt(prompt))] - - def parse_classification_result(self, result: str, branch: List[Dict]) -> Dict[str, Any]: - """解析分类结果""" - - other_branch = self.find_other_branch(branch) - normal_intents = [ - b - for b in branch - if not b.get('isOther') - ] - - def get_branch_by_id(category_id: int): - if category_id == 0: - return other_branch - elif 1 <= category_id <= len(normal_intents): - return normal_intents[category_id - 1] - return None - - try: - result_json = json.loads(result) - classification_id = result_json.get('classificationId') - # 如果是 0 ，返回其他分支 - matched_branch = get_branch_by_id(classification_id) - if matched_branch: - return matched_branch - - except Exception as e: - # json 解析失败，re 提取 - numbers = re.findall(r'"classificationId":\s*(\d+)', result) - if numbers: - classification_id = int(numbers[0]) - - matched_branch = get_branch_by_id(classification_id) - if matched_branch: - return matched_branch - - # 如果都解析失败，返回“other” - return other_branch or (normal_intents[0] if normal_intents else {'id': 'unknown', 'content': 'unknown'}) - - def parse_result_reason(self, result: str): - """解析分类的原因""" - try: - result_json = json.loads(result) - return result_json.get('reason', '') - except Exception as e: - reason_patterns = [ - r'"reason":\s*"([^"]*)"', # 标准格式 - r'"reason":\s*"([^"]*)', # 缺少结束引号 - r'"reason":\s*([^,}\n]*)', # 没有引号包围的内容 - ] - for pattern in reason_patterns: - match = re.search(pattern, result, re.DOTALL) - if match: - reason = match.group(1).strip() - # 清理可能的尾部字符 - reason = re.sub(r'["\s]*$', '', reason) - return reason - - return '' - - def find_other_branch(self, branch: List[Dict]) -> Dict[str, Any] | None: - """查找其他分支""" - for b in branch: - if b.get('isOther'): - return b - return None - - def get_details(self, index: int, **kwargs): - """获取节点执行详情""" - return { - 'name': self.node.properties.get('stepName'), - 'index': index, - 'run_time': self.context.get('run_time'), - 'system': self.context.get('system'), - 'history_message': [ - {'content': message.content, 'role': message.type} - for message in (self.context.get('history_message') or []) - ], - 'user_input': self.context.get('user_input'), - 'answer': self.context.get('answer'), - 'branch_id': self.context.get('branch_id'), - 'category': self.context.get('category'), - 'type': self.node.type, - 'message_tokens': self.context.get('message_tokens'), - 'answer_tokens': self.context.get('answer_tokens'), - 'status': self.status, - 'err_message': self.err_message, - 'enableException': self.node.properties.get('enableException'), - } diff --git a/apps/application/flow/step_node/intent_node/impl/prompt_template.py b/apps/application/flow/step_node/intent_node/impl/prompt_template.py deleted file mode 100644 index 1bcfd61743e..00000000000 --- a/apps/application/flow/step_node/intent_node/impl/prompt_template.py +++ /dev/null @@ -1,29 +0,0 @@ - - -PROMPT_TEMPLATE = """# Role -You are an intention classification expert, good at being able to judge which classification the user's input belongs to. - -## Skills -Skill 1: Clearly determine which of the following intention classifications the user's input belongs to. -Intention classification list: -{classification_list} - -Note: -- Please determine the match between the user's input content and the Intention classification list content, without judging or categorizing the match with the classification ID. -- **When classifying, you must give higher weight to the context and intent continuity shown in the historical conversation. Do not rely solely on the literal meaning of the current input; instead, prioritize the most consistent classification with the previous dialogue flow.** - -## User Input -{user_input} - -## Reply requirements -- The answer must be returned in JSON format. -- Strictly ensure that the output is in a valid JSON format. -- Do not add prefix ```json or suffix ``` -- The answer needs to include the following fields such as: -{{ -"classificationId": 0, -"reason": "" -}} - -## Limit -- Please do not reply in text.""" diff --git a/apps/application/flow/step_node/knowledge_write_node/__init__.py b/apps/application/flow/step_node/knowledge_write_node/__init__.py deleted file mode 100644 index ea50569d563..00000000000 --- a/apps/application/flow/step_node/knowledge_write_node/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：niu - @file： __init__.py.py - @date：2025/11/13 11:17 - @desc: -""" \ No newline at end of file diff --git a/apps/application/flow/step_node/knowledge_write_node/i_knowledge_write_node.py b/apps/application/flow/step_node/knowledge_write_node/i_knowledge_write_node.py deleted file mode 100644 index 2f5349fa613..00000000000 --- a/apps/application/flow/step_node/knowledge_write_node/i_knowledge_write_node.py +++ /dev/null @@ -1,43 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：niu - @file： i_knowledge_write_node.py - @date：2025/11/13 11:19 - @desc: -""" -from typing import Type - -from django.utils.translation import gettext_lazy as _ -from rest_framework import serializers - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import INode, NodeResult - - -class KnowledgeWriteNodeParamSerializer(serializers.Serializer): - document_list = serializers.ListField(required=True, child=serializers.CharField(required=True), allow_null=True, - label=_('document list')) - - -class IKnowledgeWriteNode(INode): - - def save_context(self, details, workflow_manage): - pass - - def get_node_params_serializer_class(self) -> Type[serializers.Serializer]: - return KnowledgeWriteNodeParamSerializer - - def _run(self): - documents = self.workflow_manage.get_reference_field( - self.node_params_serializer.data.get('document_list')[0], - self.node_params_serializer.data.get('document_list')[1:], - ) - - return self.execute(**self.node_params_serializer.data, **self.flow_params_serializer.data, documents=documents) - - def execute(self, documents, user_id, **kwargs) -> NodeResult: - pass - - type = 'knowledge-write-node' - support = [WorkflowMode.KNOWLEDGE, WorkflowMode.KNOWLEDGE_LOOP] diff --git a/apps/application/flow/step_node/knowledge_write_node/impl/__init__.py b/apps/application/flow/step_node/knowledge_write_node/impl/__init__.py deleted file mode 100644 index 077d7432575..00000000000 --- a/apps/application/flow/step_node/knowledge_write_node/impl/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：niu - @file： __init__.py.py - @date：2025/11/13 11:18 - @desc: -""" diff --git a/apps/application/flow/step_node/knowledge_write_node/impl/base_knowledge_write_node.py b/apps/application/flow/step_node/knowledge_write_node/impl/base_knowledge_write_node.py deleted file mode 100644 index 163361f3478..00000000000 --- a/apps/application/flow/step_node/knowledge_write_node/impl/base_knowledge_write_node.py +++ /dev/null @@ -1,380 +0,0 @@ -# coding=utf-8 -""" -@project: MaxKB -@Author：niu -@file： base_knowledge_write_node.py -@date：2025/11/13 11:19 -@desc: -""" - -from functools import reduce -from typing import Any, Dict, List - -import uuid_utils.compat as uuid -from common.chunk import text_to_chunk -from common.utils.common import bulk_create_in_batches, filter_special_character -from django.db.models import QuerySet -from django.db.models.aggregates import Max -from django.utils.translation import gettext_lazy as _ -from knowledge.models import ( - ContentOrigin, - Document, - DocumentResourceType, - DocumentTag, - File, - FileSourceType, - KnowledgeType, - Paragraph, - Problem, - ProblemParagraphMapping, - Tag, -) -from knowledge.serializers.common import ProblemParagraphManage, ProblemParagraphObject -from knowledge.serializers.document import DocumentSerializers -from knowledge.serializers.document_strategy import DocumentStrategySerializer -from knowledge.services.document_strategy import ( - document_source_hash, - normalize_document_strategy, - strategy_hashes, -) -from knowledge.services.incremental_sync import prepare_remote_paragraphs -from rest_framework import serializers - -from application.flow.i_step_node import NodeResult -from application.flow.step_node.knowledge_write_node.i_knowledge_write_node import IKnowledgeWriteNode - - -class ParagraphInstanceSerializer(serializers.Serializer): - content = serializers.CharField( - required=True, label=_("content"), max_length=102400, min_length=1, allow_null=True, allow_blank=True - ) - title = serializers.CharField( - required=False, max_length=256, label=_("section title"), allow_null=True, allow_blank=True - ) - problem_list = serializers.ListField(required=False, child=serializers.CharField(required=False, allow_blank=True)) - is_active = serializers.BooleanField(required=False, label=_("Is active")) - chunks = serializers.ListField(required=False, child=serializers.CharField(required=True)) - - -class TagInstanceSerializer(serializers.Serializer): - key = serializers.CharField(required=True, max_length=64, label=_("Tag Key")) - value = serializers.CharField(required=True, max_length=128, label=_("Tag Value")) - - -class KnowledgeWriteParamSerializer(serializers.Serializer): - name = serializers.CharField( - required=True, label=_("document name"), max_length=128, min_length=1, source=_("document name") - ) - meta = serializers.DictField(required=False) - tags = serializers.ListField(required=False, label=_("Tags"), child=TagInstanceSerializer()) - paragraphs = ParagraphInstanceSerializer(required=False, many=True, allow_null=True) - source_file_id = serializers.UUIDField(required=False, allow_null=True) - user_id = serializers.UUIDField(required=False, allow_null=True) - doc_strategy = DocumentStrategySerializer(required=False, allow_null=True) - - -def convert_uuid_to_str(obj): - if isinstance(obj, dict): - return {k: convert_uuid_to_str(v) for k, v in obj.items()} - elif isinstance(obj, list): - return [convert_uuid_to_str(i) for i in obj] - elif isinstance(obj, uuid.UUID): - return str(obj) - else: - return obj - - -def link_file(source_file_id, document_id): - if source_file_id is None: - return - source_file = QuerySet(File).filter(id=source_file_id).first() - if source_file: - file_content = source_file.get_bytes() - - new_file = File( - id=uuid.uuid7(), - file_name=source_file.file_name, - file_size=source_file.file_size, - source_type=FileSourceType.DOCUMENT, - source_id=document_id, # 更新为当前知识库ID - meta=source_file.meta.copy() if source_file.meta else {}, - ) - - # 保存文件内容和元数据 - new_file.save(file_content) - - -def get_paragraph_problem_model(knowledge_id: str, document_id: str, instance: Dict): - content = filter_special_character(instance.get("content")) - paragraph = Paragraph( - id=uuid.uuid7(), - document_id=document_id, - content=content, - knowledge_id=knowledge_id, - title=instance.get("title") if "title" in instance else "", - chunks=[ - filter_special_character(c) - for c in ( - instance.get("chunks") - if "chunks" in instance - else text_to_chunk(content, instance.get("child_length", 256)) - ) - ], - origin=instance.get("origin", ContentOrigin.SYNCED), - source_key=instance.get("source_key", ""), - source_hash=instance.get("source_hash", ""), - source_snapshot=instance.get("source_snapshot") - or { - "title": instance.get("title") or "", - "content": content, - }, - source_updated_at=instance.get("source_updated_at"), - ) - - problem_paragraph_object_list = [ - ProblemParagraphObject(knowledge_id, document_id, str(paragraph.id), problem) - for problem in (instance.get("problem_list") if "problem_list" in instance else []) - ] - - return { - "paragraph": paragraph, - "problem_paragraph_object_list": problem_paragraph_object_list, - } - - -def get_paragraph_model(document_model, paragraph_list: List): - knowledge_id = document_model.knowledge_id - paragraph_model_dict_list = [ - get_paragraph_problem_model(knowledge_id, document_model.id, paragraph) for paragraph in paragraph_list - ] - - paragraph_model_list = [] - problem_paragraph_object_list = [] - for paragraphs in paragraph_model_dict_list: - paragraph = paragraphs.get("paragraph") - for problem_model in paragraphs.get("problem_paragraph_object_list"): - problem_paragraph_object_list.append(problem_model) - paragraph_model_list.append(paragraph) - - return { - "document": document_model, - "paragraph_model_list": paragraph_model_list, - "problem_paragraph_object_list": problem_paragraph_object_list, - } - - -def get_document_paragraph_model(knowledge_id: str, instance: Dict): - source_meta = {"source_file_id": instance.get("source_file_id")} if instance.get("source_file_id") else {} - meta = {**instance.get("meta"), **source_meta} if instance.get("meta") is not None else source_meta - meta = {**convert_uuid_to_str(meta), "allow_download": True} - - strategy = normalize_document_strategy(instance.get("doc_strategy")) - normalized_paragraphs = prepare_remote_paragraphs( - [ - { - **paragraph, - "content": filter_special_character(paragraph.get("content")), - "origin": ContentOrigin.SYNCED, - "child_length": strategy["split"]["child_length"], - } - for paragraph in instance.get("paragraphs", []) - ] - ) - document_model = Document( - **{ - "knowledge_id": knowledge_id, - "id": uuid.uuid7(), - "name": instance.get("name"), - "char_length": reduce(lambda x, y: x + y, [len(p.get("content")) for p in normalized_paragraphs], 0), - "meta": meta, - "type": instance.get("type") if instance.get("type") is not None else KnowledgeType.WORKFLOW, - "resource_type": DocumentResourceType.DOCUMENT, - "doc_strategy": strategy, - "source_hash": document_source_hash(normalized_paragraphs), - "user_id": instance.get("user_id"), - **strategy_hashes(strategy), - } - ) - - return get_paragraph_model(document_model, normalized_paragraphs) - - -def save_knowledge_tags(knowledge_id: str, tags: List[Dict[str, Any]]): - existed_tags_dict = { - (key, value): str(tag_id) - for key, value, tag_id in QuerySet(Tag).filter(knowledge_id=knowledge_id).values_list("key", "value", "id") - } - - tag_model_list = [] - new_tag_dict = {} - for tag in tags: - key = tag.get("key") - value = tag.get("value") - - if (key, value) not in existed_tags_dict: - tag_model = Tag(id=uuid.uuid7(), knowledge_id=knowledge_id, key=key, value=value) - tag_model_list.append(tag_model) - new_tag_dict[(key, value)] = str(tag_model.id) - - if tag_model_list: - Tag.objects.bulk_create(tag_model_list) - - all_tag_dict = {**existed_tags_dict, **new_tag_dict} - - return all_tag_dict, new_tag_dict - - -def batch_add_document_tag(document_tag_map: Dict[str, List[str]]): - """ - 批量添加文档-标签关联 - document_tag_map: {document_id: [tag_id1, tag_id2, ...]} - """ - all_document_ids = list(document_tag_map.keys()) - all_tag_ids = list(set(tag_id for tag_ids in document_tag_map.values() for tag_id in tag_ids)) - - # 查询已存在的文档-标签关联 - existed_relations = set( - QuerySet(DocumentTag) - .filter(document_id__in=all_document_ids, tag_id__in=all_tag_ids) - .values_list("document_id", "tag_id") - ) - - new_relations = [ - DocumentTag( - id=uuid.uuid7(), - document_id=doc_id, - tag_id=tag_id, - ) - for doc_id, tag_ids in document_tag_map.items() - for tag_id in tag_ids - if (doc_id, tag_id) not in existed_relations - ] - - if new_relations: - QuerySet(DocumentTag).bulk_create(new_relations) - - -class BaseKnowledgeWriteNode(IKnowledgeWriteNode): - def save_context(self, details, workflow_manage): - self.context["exception_message"] = details.get("err_message") - - def save(self, document_list, user_id): - serializer = KnowledgeWriteParamSerializer(data=document_list, many=True) - serializer.is_valid(raise_exception=True) - document_list = serializer.data - - knowledge_id = self.workflow_params.get("knowledge_id") - workspace_id = self.workflow_params.get("workspace_id") - - document_model_list = [] - paragraph_model_list = [] - problem_paragraph_object_list = [] - # 所有标签 - knowledge_tag_list = [] - # 文档标签映射关系 - document_tags_map = {} - knowledge_tag_dict = {} - - for document in document_list: - document["user_id"] = user_id - document_paragraph_dict_model = get_document_paragraph_model(knowledge_id, document) - document_instance = document_paragraph_dict_model.get("document") - link_file(document.get("source_file_id"), document_instance.id) - document_model_list.append(document_instance) - # 收集标签 - single_document_tag_list = document.get("tags", []) - # 去重传入的标签 - for tag in single_document_tag_list: - tag_key = (tag["key"], tag["value"]) - if tag_key not in knowledge_tag_dict: - knowledge_tag_dict[tag_key] = tag - - if single_document_tag_list: - document_tags_map[str(document_instance.id)] = single_document_tag_list - - for paragraph in document_paragraph_dict_model.get("paragraph_model_list"): - paragraph_model_list.append(paragraph) - for problem_paragraph_object in document_paragraph_dict_model.get("problem_paragraph_object_list"): - problem_paragraph_object_list.append(problem_paragraph_object) - knowledge_tag_list = list(knowledge_tag_dict.values()) - # 保存所有文档中含有的标签到知识库 - if knowledge_tag_list: - all_tag_dict, new_tag_dict = save_knowledge_tags(knowledge_id, knowledge_tag_list) - # 构建文档-标签ID映射 - document_tag_id_map = {} - # 为每个文档添加其对应的标签 - for doc_id, doc_tags in document_tags_map.items(): - doc_tag_ids = [ - all_tag_dict[(tag.get("key"), tag.get("value"))] - for tag in doc_tags - if (tag.get("key"), tag.get("value")) in all_tag_dict - ] - if doc_tag_ids: - document_tag_id_map[doc_id] = doc_tag_ids - if document_tag_id_map: - batch_add_document_tag(document_tag_id_map) - - problem_model_list, problem_paragraph_mapping_list = ProblemParagraphManage( - problem_paragraph_object_list, knowledge_id - ).to_problem_model_list() - - QuerySet(Document).bulk_create(document_model_list) if len(document_model_list) > 0 else None - - if len(paragraph_model_list) > 0: - for document in document_model_list: - max_position = ( - Paragraph.objects.filter(document_id=document.id).aggregate(max_position=Max("position"))[ - "max_position" - ] - or 0 - ) - sub_list = [p for p in paragraph_model_list if p.document_id == document.id] - for i, paragraph in enumerate(sub_list): - paragraph.position = max_position + i + 1 - QuerySet(Paragraph).bulk_create(sub_list if len(sub_list) > 0 else []) - - bulk_create_in_batches(Problem, problem_model_list, batch_size=1000) - - bulk_create_in_batches(ProblemParagraphMapping, problem_paragraph_mapping_list, batch_size=1000) - - return document_model_list, knowledge_id, workspace_id - - @staticmethod - def post_embedding(document_model_list, knowledge_id, workspace_id): - for document in document_model_list: - DocumentSerializers.Operate( - data={"knowledge_id": knowledge_id, "document_id": document.id, "workspace_id": workspace_id} - ).refresh() - - def execute(self, documents, user_id, **kwargs) -> NodeResult: - - document_model_list, knowledge_id, workspace_id = self.save(documents, user_id) - self.post_embedding(document_model_list, knowledge_id, workspace_id) - - write_content_list = [ - { - "name": document.get("name"), - "paragraphs": [ - { - "title": p.get("title"), - "content": p.get("content"), - } - for p in document.get("paragraphs")[0:5] - ], - } - for document in documents - ] - - return NodeResult({"write_content": write_content_list}, {}) - - def get_details(self, index: int, **kwargs): - return { - "name": self.node.properties.get("stepName"), - "index": index, - "run_time": self.context.get("run_time"), - "type": self.node.type, - "write_content": self.context.get("write_content"), - "status": self.status, - "err_message": self.err_message, - "enableException": self.node.properties.get("enableException"), - } diff --git a/apps/application/flow/step_node/loop_break_node/__init__.py b/apps/application/flow/step_node/loop_break_node/__init__.py deleted file mode 100644 index ee45b3ee837..00000000000 --- a/apps/application/flow/step_node/loop_break_node/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎虎 - @file： __init__.py.py - @date：2025/9/15 12:08 - @desc: -""" -from .impl import * diff --git a/apps/application/flow/step_node/loop_break_node/i_loop_break_node.py b/apps/application/flow/step_node/loop_break_node/i_loop_break_node.py deleted file mode 100644 index 07edf227b53..00000000000 --- a/apps/application/flow/step_node/loop_break_node/i_loop_break_node.py +++ /dev/null @@ -1,41 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎虎 - @file： i_loop_break_node.py - @date：2025/9/15 12:14 - @desc: -""" -from typing import Type - -from django.utils.translation import gettext_lazy as _ -from rest_framework import serializers - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import INode -from application.flow.i_step_node import NodeResult - - -class ConditionSerializer(serializers.Serializer): - compare = serializers.CharField(required=True, label=_("Comparator")) - value = serializers.CharField(required=False, allow_null=True, allow_blank=True, label=_("value")) - field = serializers.ListField(required=True, label=_("Fields")) - - -class LoopBreakNodeSerializer(serializers.Serializer): - condition = serializers.CharField(required=True, label=_("Condition or|and")) - condition_list = ConditionSerializer(many=True) - - -class ILoopBreakNode(INode): - type = 'loop-break-node' - support = [WorkflowMode.APPLICATION_LOOP, WorkflowMode.KNOWLEDGE_LOOP, WorkflowMode.TOOL_LOOP] - - def get_node_params_serializer_class(self) -> Type[serializers.Serializer]: - return LoopBreakNodeSerializer - - def _run(self): - return self.execute(**self.node_params_serializer.data) - - def execute(self, condition, condition_list, **kwargs) -> NodeResult: - pass diff --git a/apps/application/flow/step_node/loop_break_node/impl/__init__.py b/apps/application/flow/step_node/loop_break_node/impl/__init__.py deleted file mode 100644 index 0ed3e008022..00000000000 --- a/apps/application/flow/step_node/loop_break_node/impl/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎虎 - @file： __init__.py.py - @date：2025/9/15 12:16 - @desc: -""" -from .base_loop_break_node import BaseLoopBreakNode diff --git a/apps/application/flow/step_node/loop_break_node/impl/base_loop_break_node.py b/apps/application/flow/step_node/loop_break_node/impl/base_loop_break_node.py deleted file mode 100644 index f82289729da..00000000000 --- a/apps/application/flow/step_node/loop_break_node/impl/base_loop_break_node.py +++ /dev/null @@ -1,47 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎虎 - @file： base_loop_break_node.py - @date：2025/9/15 12:17 - @desc: -""" -import time -from typing import Dict - -from application.flow.compare import do_assertion -from application.flow.i_step_node import NodeResult -from application.flow.step_node.loop_break_node.i_loop_break_node import ILoopBreakNode - - -def _write_context(step_variable: Dict, global_variable: Dict, node, workflow): - if step_variable.get("is_break"): - yield "BREAK" - - node.context['run_time'] = time.time() - node.context['start_time'] - - -class BaseLoopBreakNode(ILoopBreakNode): - def save_context(self, details, workflow_manage): - self.context['exception_message'] = details.get('err_message') - - def execute(self, condition, condition_list, **kwargs) -> NodeResult: - is_break = do_assertion(self.workflow_manage, condition, condition_list) - if is_break: - self.node_params['is_result'] = True - self.context['is_break'] = is_break - return NodeResult({'is_break': is_break}, {}, - _write_context=_write_context, - _is_interrupt=lambda n, v, w: is_break) - - def get_details(self, index: int, **kwargs): - return { - 'name': self.node.properties.get('stepName'), - "index": index, - 'is_break': self.context.get('is_break'), - 'run_time': self.context.get('run_time'), - 'type': self.node.type, - 'status': self.status, - 'err_message': self.err_message, - 'enableException': self.node.properties.get('enableException'), - } diff --git a/apps/application/flow/step_node/loop_continue_node/__init__.py b/apps/application/flow/step_node/loop_continue_node/__init__.py deleted file mode 100644 index 9f7f1729d5c..00000000000 --- a/apps/application/flow/step_node/loop_continue_node/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎虎 - @file： __init__.py.py - @date：2025/9/15 12:08 - @desc: -""" -from .impl import * \ No newline at end of file diff --git a/apps/application/flow/step_node/loop_continue_node/i_loop_continue_node.py b/apps/application/flow/step_node/loop_continue_node/i_loop_continue_node.py deleted file mode 100644 index 00b6aa04c39..00000000000 --- a/apps/application/flow/step_node/loop_continue_node/i_loop_continue_node.py +++ /dev/null @@ -1,40 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎虎 - @file： i_loop_continue_node.py - @date：2025/9/15 12:13 - @desc: -""" -from typing import Type - -from django.utils.translation import gettext_lazy as _ -from rest_framework import serializers - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import INode, NodeResult - - -class ConditionSerializer(serializers.Serializer): - compare = serializers.CharField(required=True, label=_("Comparator")) - value = serializers.CharField(required=True, label=_("value")) - field = serializers.ListField(required=True, label=_("Fields")) - - -class LoopContinueNodeSerializer(serializers.Serializer): - condition = serializers.CharField(required=True, label=_("Condition or|and")) - condition_list = ConditionSerializer(many=True) - - -class ILoopContinueNode(INode): - type = 'loop-continue-node' - support = [WorkflowMode.APPLICATION_LOOP, WorkflowMode.KNOWLEDGE_LOOP, WorkflowMode.TOOL_LOOP] - - def get_node_params_serializer_class(self) -> Type[serializers.Serializer]: - return LoopContinueNodeSerializer - - def _run(self): - return self.execute(**self.node_params_serializer.data) - - def execute(self, condition, condition_list, **kwargs) -> NodeResult: - pass diff --git a/apps/application/flow/step_node/loop_continue_node/impl/__init__.py b/apps/application/flow/step_node/loop_continue_node/impl/__init__.py deleted file mode 100644 index 3aca2f827de..00000000000 --- a/apps/application/flow/step_node/loop_continue_node/impl/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎虎 - @file： __init__.py.py - @date：2025/9/15 12:13 - @desc: -""" -from .base_loop_continue_node import BaseLoopContinueNode \ No newline at end of file diff --git a/apps/application/flow/step_node/loop_continue_node/impl/base_loop_continue_node.py b/apps/application/flow/step_node/loop_continue_node/impl/base_loop_continue_node.py deleted file mode 100644 index 3c0393217c5..00000000000 --- a/apps/application/flow/step_node/loop_continue_node/impl/base_loop_continue_node.py +++ /dev/null @@ -1,35 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎虎 - @file： base_loop_continue_node.py - @date：2025/9/15 12:13 - @desc: -""" -from application.flow.compare import do_assertion -from application.flow.i_step_node import NodeResult -from application.flow.step_node.loop_continue_node.i_loop_continue_node import ILoopContinueNode - - -class BaseLoopContinueNode(ILoopContinueNode): - def save_context(self, details, workflow_manage): - self.context['exception_message'] = details.get('err_message') - - def execute(self, condition, condition_list, **kwargs) -> NodeResult: - is_continue = do_assertion(self.workflow_manage, condition, condition_list) - self.context['is_continue'] = is_continue - if is_continue: - return NodeResult({'is_continue': is_continue, 'branch_id': 'continue'}, {}) - return NodeResult({'is_continue': is_continue}, {}) - - def get_details(self, index: int, **kwargs): - return { - 'name': self.node.properties.get('stepName'), - "index": index, - "is_continue": self.context.get('is_continue'), - 'run_time': self.context.get('run_time'), - 'type': self.node.type, - 'status': self.status, - 'err_message': self.err_message, - 'enableException': self.node.properties.get('enableException'), - } diff --git a/apps/application/flow/step_node/loop_node/__init__.py b/apps/application/flow/step_node/loop_node/__init__.py deleted file mode 100644 index a5f59372be7..00000000000 --- a/apps/application/flow/step_node/loop_node/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎 - @file： __init__.py - @date：2025/3/11 18:24 - @desc: -""" -from .impl import * diff --git a/apps/application/flow/step_node/loop_node/i_loop_node.py b/apps/application/flow/step_node/loop_node/i_loop_node.py deleted file mode 100644 index 527487f5412..00000000000 --- a/apps/application/flow/step_node/loop_node/i_loop_node.py +++ /dev/null @@ -1,58 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎 - @file： i_loop_node.py - @date：2025/3/11 18:19 - @desc: -""" -from typing import Type - -from django.utils.translation import gettext_lazy as _ -from rest_framework import serializers - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import INode, NodeResult -from common.exception.app_exception import AppApiException - - -class ILoopNodeSerializer(serializers.Serializer): - loop_type = serializers.CharField(required=True, label=_("loop_type")) - array = serializers.ListField(required=False, allow_null=True, - label=_("array")) - number = serializers.IntegerField(required=False, allow_null=True, - label=_("number")) - loop_body = serializers.DictField(required=True, label="循环体") - - def is_valid(self, *, raise_exception=False): - super().is_valid(raise_exception=True) - loop_type = self.data.get('loop_type') - if loop_type == 'ARRAY': - array = self.data.get('array') - if array is None or len(array) == 0: - message = _('{field}, this field is required.').format(field='array') - raise AppApiException(500, message) - elif loop_type == 'NUMBER': - number = self.data.get('number') - if number is None: - message = _('{field}, this field is required.').format(field='number') - raise AppApiException(500, message) - - -class ILoopNode(INode): - type = 'loop-node' - support = [WorkflowMode.APPLICATION, WorkflowMode.KNOWLEDGE, WorkflowMode.TOOL] - - def get_node_params_serializer_class(self) -> Type[serializers.Serializer]: - return ILoopNodeSerializer - - def _run(self): - array = self.node_params_serializer.data.get('array') - if self.node_params_serializer.data.get('loop_type') == 'ARRAY': - array = self.workflow_manage.get_reference_field( - array[0], - array[1:]) - return self.execute(**{**self.node_params_serializer.data, "array": array}, **self.flow_params_serializer.data) - - def execute(self, loop_type, array, number, loop_body, **kwargs) -> NodeResult: - pass diff --git a/apps/application/flow/step_node/loop_node/impl/__init__.py b/apps/application/flow/step_node/loop_node/impl/__init__.py deleted file mode 100644 index 3cd082322a1..00000000000 --- a/apps/application/flow/step_node/loop_node/impl/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎 - @file： __init__.py.py - @date：2025/3/11 18:24 - @desc: -""" -from .base_loop_node import BaseLoopNode diff --git a/apps/application/flow/step_node/loop_node/impl/base_loop_node.py b/apps/application/flow/step_node/loop_node/impl/base_loop_node.py deleted file mode 100644 index e3f3cfa4e31..00000000000 --- a/apps/application/flow/step_node/loop_node/impl/base_loop_node.py +++ /dev/null @@ -1,332 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎 - @file： base_loop_node.py - @date：2025/3/11 18:24 - @desc: -""" -import time -import uuid -from typing import Dict, List - -from django.utils.translation import gettext as _ - -from application.flow.common import Answer, WorkflowMode -from application.flow.i_step_node import NodeResult, WorkFlowPostHandler, INode -from application.flow.step_node.loop_node.i_loop_node import ILoopNode -from application.flow.tools import Reasoning -from application.models import ChatRecord -from common.handle.impl.response.loop_to_response import LoopToResponse -from maxkb.const import CONFIG - -max_loop_count = int(CONFIG.get("WORKFLOW_LOOP_NODE_MAX_LOOP_COUNT", 500)) - - -def _is_interrupt_exec(node, node_variable: Dict, workflow_variable: Dict): - return node.context.get('is_interrupt_exec', False) - - -def _write_context(node_variable: Dict, workflow_variable: Dict, node: INode, workflow, answer: str, - reasoning_content: str): - node.context['answer'] = answer - node.context['run_time'] = time.time() - node.context['start_time'] - node.context['reasoning_content'] = reasoning_content - if workflow.is_result(node, NodeResult(node_variable, workflow_variable)): - node.answer_text = answer - - -def write_context_stream(node_variable: Dict, workflow_variable: Dict, node: INode, workflow): - """ - 写入上下文数据 (流式) - @param node_variable: 节点数据 - @param workflow_variable: 全局数据 - @param node: 节点 - @param workflow: 工作流管理器 - """ - - response = node_variable.get('result') - workflow_manage = node_variable.get('workflow_manage') - answer = '' - reasoning_content = '' - for chunk in response: - content_chunk = chunk.get('content', '') - reasoning_content_chunk = chunk.get('reasoning_content', '') - reasoning_content += reasoning_content_chunk - answer += content_chunk - yield {'content': content_chunk, - 'reasoning_content': reasoning_content_chunk} - runtime_details = workflow_manage.get_runtime_details() - _write_context(node_variable, workflow_variable, node, workflow, answer, reasoning_content) - - -def write_context(node_variable: Dict, workflow_variable: Dict, node: INode, workflow): - """ - 写入上下文数据 - @param node_variable: 节点数据 - @param workflow_variable: 全局数据 - @param node: 节点实例对象 - @param workflow: 工作流管理器 - """ - response = node_variable.get('result') - model_setting = node.context.get('model_setting', - {'reasoning_content_enable': False, 'reasoning_content_end': '', - 'reasoning_content_start': ''}) - reasoning = Reasoning(model_setting.get('reasoning_content_start'), model_setting.get('reasoning_content_end')) - reasoning_result = reasoning.get_reasoning_content(response) - reasoning_result_end = reasoning.get_end_reasoning_content() - content = reasoning_result.get('content') + reasoning_result_end.get('content') - if 'reasoning_content' in response.response_metadata: - reasoning_content = response.response_metadata.get('reasoning_content', '') - else: - reasoning_content = reasoning_result.get('reasoning_content') + reasoning_result_end.get('reasoning_content') - _write_context(node_variable, workflow_variable, node, workflow, content, reasoning_content) - - -def get_answer_list(instance, child_node_node_dict, runtime_node_id): - answer_list = instance.get_record_answer_list() - for a in answer_list: - _v = child_node_node_dict.get(a.get('runtime_node_id')) - if _v: - a['runtime_node_id'] = runtime_node_id - a['child_node'] = _v - return answer_list - - -def insert_or_replace(arr, index, value): - if index < len(arr): - arr[index] = value # 替换 - else: - # 在末尾插入足够多的None，然后替换最后一个 - arr.extend([None] * (index - len(arr) + 1)) - arr[index] = value - return arr - - -def generate_loop_number(number: int): - def i(current_index: int): - return iter([(index, index) for index in range(current_index, number)]) - - return i - - -def generate_loop_array(array): - def i(current_index: int): - return iter([(array[index], index) for index in range(current_index, len(array))]) - - return i - - -def generate_while_loop(current_index: int): - index = current_index - while True: - yield index, index - index += 1 - - -def loop(workflow_manage_new_instance, node: INode, generate_loop): - loop_global_data = {} - break_outer = False - is_interrupt_exec = False - loop_node_data = node.context.get('loop_node_data') or [] - loop_answer_data = node.context.get("loop_answer_data") or [] - start_index = node.context.get("current_index") or 0 - current_index = start_index - node_params = node.node_params - start_node_id = node_params.get('child_node', {}).get('runtime_node_id') - loop_type = node_params.get('loop_type') - start_node_data = None - chat_record = None - child_node = None - if start_node_id: - chat_record_id = node_params.get('child_node', {}).get('chat_record_id') - child_node = node_params.get('child_node', {}).get('child_node') - start_node_data = node_params.get('node_data') - chat_record = ChatRecord(id=chat_record_id, answer_text_list=[], answer_text='', - details=loop_node_data[current_index]) - - for item, index in generate_loop(current_index): - if 0 < max_loop_count <= index - start_index and loop_type == 'LOOP': - raise Exception(_('Exceeding the maximum number of cycles')) - """ - 指定次数循环 - @return: - """ - instance = workflow_manage_new_instance({'index': index, 'item': item}, loop_global_data, start_node_id, - start_node_data, chat_record, child_node) - response = instance.stream() - answer = '' - current_index = index - reasoning_content = '' - child_node_node_dict = {} - for chunk in response: - if chunk.get('node_type') == 'loop-break-node' and chunk.get('content', '') == 'BREAK': - break_outer = True - continue - child_node = chunk.get('child_node') - runtime_node_id = chunk.get('runtime_node_id', '') - chat_record_id = chunk.get('chat_record_id', '') - child_node_node_dict[runtime_node_id] = { - 'runtime_node_id': runtime_node_id, - 'chat_record_id': chat_record_id, - 'child_node': child_node} - content_chunk = (chunk.get('content', '') or '') - reasoning_content_chunk = (chunk.get('reasoning_content', '') or '') - if chunk.get('real_node_id'): - chunk['real_node_id'] = chunk['real_node_id'] + '__' + node.runtime_node_id + '__' + str(index) - reasoning_content += reasoning_content_chunk - answer += content_chunk - yield chunk - if chunk.get('node_status', "SUCCESS") == 'ERROR': - insert_or_replace(loop_node_data, index, instance.get_runtime_details()) - insert_or_replace(loop_answer_data, index, - get_answer_list(instance, child_node_node_dict, node.runtime_node_id)) - node.context['is_interrupt_exec'] = is_interrupt_exec - node.context['loop_node_data'] = loop_node_data - node.context['loop_answer_data'] = loop_answer_data - node.context["index"] = current_index - node.context["item"] = current_index - node.status = 500 - node.err_message = chunk.get('content') - return - node_type = chunk.get('node_type') - if node_type == 'form-node': - break_outer = True - is_interrupt_exec = True - start_node_id = None - start_node_data = None - chat_record = None - child_node = None - insert_or_replace(loop_node_data, index, instance.get_runtime_details()) - insert_or_replace(loop_answer_data, index, - get_answer_list(instance, child_node_node_dict, node.runtime_node_id)) - instance._cleanup() - if break_outer: - break - if instance.is_the_task_interrupted(): - break - node.context['is_interrupt_exec'] = is_interrupt_exec - node.context['loop_node_data'] = loop_node_data - node.context['loop_answer_data'] = loop_answer_data - node.context["index"] = current_index - node.context["item"] = current_index - node.context['run_time'] = time.time() - node.context.get("start_time") - - -def get_tokens(loop_node_data): - message_tokens = 0 - answer_tokens = 0 - for details in (loop_node_data or {}): - message_tokens += sum([row.get('message_tokens') or 0 for row in details.values() if - 'message_tokens' in row and row.get('message_tokens') is not None]) - answer_tokens += sum([row.get('answer_tokens') or 0 for row in details.values() if - 'answer_tokens' in row and row.get('answer_tokens') is not None]) - return {'message_tokens': message_tokens, 'answer_tokens': answer_tokens} - - -def get_write_context(loop_type, array, number, loop_body): - def inner_write_context(node_variable: Dict, workflow_variable: Dict, node: INode, workflow): - if loop_type == 'ARRAY': - return loop(node_variable['workflow_manage_new_instance'], node, generate_loop_array(array)) - if loop_type == 'LOOP': - return loop(node_variable['workflow_manage_new_instance'], node, generate_while_loop) - return loop(node_variable['workflow_manage_new_instance'], node, generate_loop_number(number)) - - return inner_write_context - - -class LoopWorkFlowPostHandler(WorkFlowPostHandler): - def handler(self, workflow): - pass - - -class BaseLoopNode(ILoopNode): - def save_context(self, details, workflow_manage): - self.context['loop_context_data'] = details.get('loop_context_data') - self.context['loop_answer_data'] = details.get('loop_answer_data') - self.context['loop_node_data'] = details.get('loop_node_data') - self.context['result'] = details.get('result') - self.context['params'] = details.get('params') - self.context['run_time'] = details.get('run_time') - self.context['index'] = details.get('current_index') - self.context['item'] = details.get('current_item') - for key, value in (details.get('loop_context_data') or {}).items(): - self.context[key] = value - self.answer_text = "" - - def get_answer_list(self) -> List[Answer] | None: - result = [] - for answer_list in (self.context.get("loop_answer_data") or []): - for a in answer_list: - if isinstance(a, dict): - result.append(Answer(**a)) - - return result - - def get_loop_context(self): - return self.context - - def execute(self, loop_type, array, number, loop_body, **kwargs) -> NodeResult: - from application.flow.loop_workflow_manage import LoopWorkflowManage, Workflow - from application.flow.knowledge_loop_workflow_manage import KnowledgeLoopWorkflowManage - from application.flow.tool_loop_workflow_manage import ToolLoopWorkflowManage - self.node_params['is_result'] = True - - def workflow_manage_new_instance(loop_data, global_data, start_node_id=None, - start_node_data=None, chat_record=None, child_node=None): - workflow_mode = {WorkflowMode.APPLICATION: WorkflowMode.APPLICATION_LOOP, - WorkflowMode.KNOWLEDGE: WorkflowMode.KNOWLEDGE_LOOP, - WorkflowMode.TOOL: WorkflowMode.TOOL_LOOP}.get( - self.workflow_manage.flow.workflow_mode) or WorkflowMode.APPLICATION - c = {WorkflowMode.APPLICATION_LOOP: LoopWorkflowManage, - WorkflowMode.KNOWLEDGE_LOOP: KnowledgeLoopWorkflowManage, - WorkflowMode.TOOL_LOOP: ToolLoopWorkflowManage}.get(workflow_mode) or LoopWorkflowManage - workflow_manage = c(Workflow.new_instance(loop_body, workflow_mode), - self.workflow_manage.params, - LoopWorkFlowPostHandler( - self.workflow_manage.work_flow_post_handler.chat_info), - self.workflow_manage, - loop_data, - self.get_loop_context, - base_to_response=LoopToResponse(), - start_node_id=start_node_id, - start_node_data=start_node_data, - chat_record=chat_record, - child_node=child_node, - is_the_task_interrupted=self.workflow_manage.is_the_task_interrupted - ) - - return workflow_manage - - return NodeResult({'workflow_manage_new_instance': workflow_manage_new_instance}, {}, - _write_context=get_write_context(loop_type, array, number, loop_body), - _is_interrupt=_is_interrupt_exec) - - def get_loop_context_data(self): - fields = self.node.properties.get('config', {}).get('fields', []) or [] - return {f.get('value'): self.context.get(f.get('value')) for f in fields if - self.context.get(f.get('value')) is not None} - - def get_details(self, index: int, **kwargs): - tokens = get_tokens(self.context.get("loop_node_data")) - return { - 'name': self.node.properties.get('stepName'), - "index": index, - "result": self.context.get('result'), - 'array': self.node_params_serializer.data.get('array'), - 'number': self.node_params_serializer.data.get('number'), - "params": self.context.get('params'), - 'run_time': self.context.get('run_time'), - 'type': self.node.type, - 'current_index': self.context.get("index"), - "current_item": self.context.get("item"), - 'loop_type': self.node_params_serializer.data.get('loop_type'), - 'status': self.status, - 'loop_context_data': self.get_loop_context_data(), - 'loop_node_data': self.context.get("loop_node_data"), - 'loop_answer_data': self.context.get("loop_answer_data"), - 'err_message': self.err_message, - 'enableException': self.node.properties.get('enableException'), - 'message_tokens': tokens.get('message_tokens') or 0, - 'answer_tokens': tokens.get('answer_tokens') or 0, - } diff --git a/apps/application/flow/step_node/loop_start_node/__init__.py b/apps/application/flow/step_node/loop_start_node/__init__.py deleted file mode 100644 index 98a1afcd904..00000000000 --- a/apps/application/flow/step_node/loop_start_node/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： __init__.py - @date：2024/6/11 15:30 - @desc: -""" -from .impl import * diff --git a/apps/application/flow/step_node/loop_start_node/i_loop_start_node.py b/apps/application/flow/step_node/loop_start_node/i_loop_start_node.py deleted file mode 100644 index 7c3ffa31413..00000000000 --- a/apps/application/flow/step_node/loop_start_node/i_loop_start_node.py +++ /dev/null @@ -1,21 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： i_start_node.py - @date：2024/6/3 16:54 - @desc: -""" -from application.flow.common import WorkflowMode -from application.flow.i_step_node import INode, NodeResult - - -class ILoopStarNode(INode): - type = 'loop-start-node' - support = [WorkflowMode.APPLICATION_LOOP, WorkflowMode.KNOWLEDGE_LOOP, WorkflowMode.TOOL_LOOP] - - def _run(self): - return self.execute(**self.flow_params_serializer.data) - - def execute(self, **kwargs) -> NodeResult: - pass diff --git a/apps/application/flow/step_node/loop_start_node/impl/__init__.py b/apps/application/flow/step_node/loop_start_node/impl/__init__.py deleted file mode 100644 index 76f972fcedb..00000000000 --- a/apps/application/flow/step_node/loop_start_node/impl/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： __init__.py - @date：2024/6/11 15:36 - @desc: -""" -from .base_start_node import BaseLoopStartStepNode diff --git a/apps/application/flow/step_node/loop_start_node/impl/base_start_node.py b/apps/application/flow/step_node/loop_start_node/impl/base_start_node.py deleted file mode 100644 index 8058e098b20..00000000000 --- a/apps/application/flow/step_node/loop_start_node/impl/base_start_node.py +++ /dev/null @@ -1,59 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： base_start_node.py - @date：2024/6/3 17:17 - @desc: -""" -from typing import Type - -from rest_framework import serializers - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import NodeResult -from application.flow.step_node.loop_start_node.i_loop_start_node import ILoopStarNode - - -class BaseLoopStartStepNode(ILoopStarNode): - def save_context(self, details, workflow_manage): - self.context['index'] = details.get('current_index') - self.context['item'] = details.get('current_item') - self.context['exception_message'] = details.get('err_message') - - def get_node_params_serializer_class(self) -> Type[serializers.Serializer]: - pass - - def execute(self, **kwargs) -> NodeResult: - """ - 开始节点 初始化全局变量 - """ - loop_params = self.workflow_manage.loop_params - node_variable = { - 'index': loop_params.get("index"), - 'item': loop_params.get("item") - } - if WorkflowMode.APPLICATION_LOOP == self.workflow_manage.flow.workflow_mode: - self.workflow_manage.chat_context = self.workflow_manage.get_chat_info().get_chat_variable() - return NodeResult(node_variable, {}) - - def get_details(self, index: int, **kwargs): - global_fields = [] - for field in self.node.properties.get('config')['globalFields']: - key = field['value'] - global_fields.append({ - 'label': field['label'], - 'key': key, - 'value': self.workflow_manage.context[key] if key in self.workflow_manage.context else '' - }) - return { - 'name': self.node.properties.get('stepName'), - "index": index, - "current_index": self.context.get('index'), - "current_item": self.context.get('item'), - 'run_time': self.context.get('run_time'), - 'type': self.node.type, - 'status': self.status, - 'err_message': self.err_message, - 'enableException': self.node.properties.get('enableException'), - } diff --git a/apps/application/flow/step_node/mcp_node/__init__.py b/apps/application/flow/step_node/mcp_node/__init__.py deleted file mode 100644 index f3feecc9ce2..00000000000 --- a/apps/application/flow/step_node/mcp_node/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# coding=utf-8 - -from .impl import * diff --git a/apps/application/flow/step_node/mcp_node/i_mcp_node.py b/apps/application/flow/step_node/mcp_node/i_mcp_node.py deleted file mode 100644 index 6dd3827d640..00000000000 --- a/apps/application/flow/step_node/mcp_node/i_mcp_node.py +++ /dev/null @@ -1,33 +0,0 @@ -# coding=utf-8 - -from typing import Type - -from django.utils.translation import gettext_lazy as _ -from rest_framework import serializers - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import INode, NodeResult - - -class McpNodeSerializer(serializers.Serializer): - mcp_servers = serializers.JSONField(required=True, label=_("Mcp servers")) - mcp_server = serializers.CharField(required=True, label=_("Mcp server")) - mcp_tool = serializers.CharField(required=True, label=_("Mcp tool")) - mcp_tool_id = serializers.CharField(required=False, label=_("Mcp tool"), allow_null=True, allow_blank=True) - mcp_source = serializers.CharField(required=False, label=_("Mcp source"), allow_blank=True, allow_null=True) - tool_params = serializers.DictField(required=True, label=_("Tool parameters")) - - -class IMcpNode(INode): - type = 'mcp-node' - support = [WorkflowMode.APPLICATION, WorkflowMode.APPLICATION_LOOP, WorkflowMode.KNOWLEDGE, - WorkflowMode.KNOWLEDGE_LOOP, WorkflowMode.TOOL, WorkflowMode.TOOL_LOOP] - - def get_node_params_serializer_class(self) -> Type[serializers.Serializer]: - return McpNodeSerializer - - def _run(self): - return self.execute(**self.node_params_serializer.data, **self.flow_params_serializer.data) - - def execute(self, mcp_servers, mcp_server, mcp_tool, mcp_tool_id, mcp_source, tool_params, **kwargs) -> NodeResult: - pass diff --git a/apps/application/flow/step_node/mcp_node/impl/__init__.py b/apps/application/flow/step_node/mcp_node/impl/__init__.py deleted file mode 100644 index 8c9a5ee197c..00000000000 --- a/apps/application/flow/step_node/mcp_node/impl/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# coding=utf-8 - -from .base_mcp_node import BaseMcpNode diff --git a/apps/application/flow/step_node/mcp_node/impl/base_mcp_node.py b/apps/application/flow/step_node/mcp_node/impl/base_mcp_node.py deleted file mode 100644 index a7bba2a6ab1..00000000000 --- a/apps/application/flow/step_node/mcp_node/impl/base_mcp_node.py +++ /dev/null @@ -1,72 +0,0 @@ -# coding=utf-8 -import asyncio -import json -from typing import List - -from django.db.models import QuerySet -from application.flow.backend.sandbox_mcp import SandboxMCPBackend - -from application.flow.i_step_node import NodeResult -from application.flow.step_node.mcp_node.i_mcp_node import IMcpNode -from tools.models import Tool -from common.utils.tool_code import ToolExecutor - - -class BaseMcpNode(IMcpNode): - def save_context(self, details, workflow_manage): - self.context["result"] = details.get("result") - self.context["tool_params"] = details.get("tool_params") - self.context["mcp_tool"] = details.get("mcp_tool") - self.context["exception_message"] = details.get("err_message") - - def execute(self, mcp_servers, mcp_server, mcp_tool, mcp_tool_id, mcp_source, tool_params, **kwargs) -> NodeResult: - if mcp_source == "referencing": - if not mcp_tool_id: - raise ValueError("MCP tool ID is required when mcp_source is 'referencing'.") - tool = QuerySet(Tool).filter(id=mcp_tool_id).first() - if not tool: - raise ValueError(f"Tool with ID {mcp_tool_id} not found.") - if not tool.is_active: - raise ValueError(f"Tool with ID {mcp_tool_id} is inactive.") - servers = json.loads(tool.code) - else: - servers = json.loads(mcp_servers) - - servers = self.handle_variables(servers) # 处理servers中的变量 - ToolExecutor().validate_mcp_transport(json.dumps(servers)) - params = json.loads(json.dumps(tool_params)) - params = self.handle_variables(params) - - backend = SandboxMCPBackend(servers) - res = asyncio.run(backend.call_tool(mcp_server, mcp_tool, params)) - return NodeResult( - {"result": [content.text for content in res.content], "tool_params": params, "mcp_tool": mcp_tool}, {} - ) - - def handle_variables(self, tool_params): - # 处理参数中的变量 - for k, v in tool_params.items(): - if type(v) == str: - tool_params[k] = self.workflow_manage.generate_prompt(tool_params[k]) - elif type(v) == dict: - self.handle_variables(v) - elif (type(v) == list) and len(v) > 0 and (type(v[0]) == str): - tool_params[k] = self.get_reference_content(v) - return tool_params - - def get_reference_content(self, fields: List[str]): - return self.workflow_manage.get_reference_field(fields[0], fields[1:]) if fields else None - - def get_details(self, index: int, **kwargs): - return { - "name": self.node.properties.get("stepName"), - "index": index, - "run_time": self.context.get("run_time"), - "status": self.status, - "err_message": self.err_message, - "type": self.node.type, - "mcp_tool": self.context.get("mcp_tool"), - "tool_params": self.context.get("tool_params"), - "result": self.context.get("result"), - "enableException": self.node.properties.get("enableException"), - } diff --git a/apps/application/flow/step_node/parameter_extraction_node/__init__.py b/apps/application/flow/step_node/parameter_extraction_node/__init__.py deleted file mode 100644 index c93d71e9ed1..00000000000 --- a/apps/application/flow/step_node/parameter_extraction_node/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎虎 - @file： __init__.py.py - @date：2025/10/13 14:56 - @desc: -""" -from .impl import * diff --git a/apps/application/flow/step_node/parameter_extraction_node/i_parameter_extraction_node.py b/apps/application/flow/step_node/parameter_extraction_node/i_parameter_extraction_node.py deleted file mode 100644 index 54c60bb096c..00000000000 --- a/apps/application/flow/step_node/parameter_extraction_node/i_parameter_extraction_node.py +++ /dev/null @@ -1,58 +0,0 @@ -# coding=utf-8 - -from typing import Type - -from django.utils.translation import gettext_lazy as _ -from rest_framework import serializers - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import INode, NodeResult - - -class VariableSplittingNodeParamsSerializer(serializers.Serializer): - input_variable = serializers.ListField(required=True, - label=_("input variable")) - - variable_list = serializers.ListField(required=True, - label=_("Split variables")) - - model_params_setting = serializers.DictField(required=False, - label=_("Model parameter settings")) - - model_id = serializers.CharField(required=False, allow_blank=True, allow_null=True, label=_("Model id")) - model_id_type = serializers.CharField(required=False, default='custom', label=_("Model id type")) - model_id_reference = serializers.ListField(required=False, child=serializers.CharField(), allow_empty=True, - label=_("Reference Field")) - - -class IParameterExtractionNode(INode): - type = 'parameter-extraction-node' - support = [WorkflowMode.APPLICATION, WorkflowMode.APPLICATION_LOOP, WorkflowMode.KNOWLEDGE, - WorkflowMode.KNOWLEDGE_LOOP, WorkflowMode.TOOL, WorkflowMode.TOOL_LOOP] - - def get_node_params_serializer_class(self) -> Type[serializers.Serializer]: - return VariableSplittingNodeParamsSerializer - - def _run(self): - model_id_type = self.node_params_serializer.data.get('model_id_type') - model_id_reference = self.node_params_serializer.data.get('model_id_reference') - model_id = self.node_params_serializer.data.get('model_id') - model_params_setting = self.node_params_serializer.data.get('model_params_setting') - # 处理引用类型 - if model_id_type == 'reference' and model_id_reference: - reference_data = self.workflow_manage.get_reference_field( - model_id_reference[0], - model_id_reference[1:], - ) - if reference_data and isinstance(reference_data, dict): - model_id = reference_data.get('model_id', model_id) - model_params_setting = reference_data.get('model_params_setting') - - input_variable = self.workflow_manage.get_reference_field( - self.node_params_serializer.data.get('input_variable')[0], - self.node_params_serializer.data.get('input_variable')[1:]) - return self.execute(input_variable, self.node_params_serializer.data['variable_list'], - model_params_setting, model_id) - - def execute(self, input_variable, variable_list, model_params_setting, model_id, **kwargs) -> NodeResult: - pass diff --git a/apps/application/flow/step_node/parameter_extraction_node/impl/__init__.py b/apps/application/flow/step_node/parameter_extraction_node/impl/__init__.py deleted file mode 100644 index a0d23a10454..00000000000 --- a/apps/application/flow/step_node/parameter_extraction_node/impl/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎虎 - @file： __init__.py.py - @date：2025/10/13 15:01 - @desc: -""" -from .base_parameter_extraction_node import * diff --git a/apps/application/flow/step_node/parameter_extraction_node/impl/base_parameter_extraction_node.py b/apps/application/flow/step_node/parameter_extraction_node/impl/base_parameter_extraction_node.py deleted file mode 100644 index 8e39084ab75..00000000000 --- a/apps/application/flow/step_node/parameter_extraction_node/impl/base_parameter_extraction_node.py +++ /dev/null @@ -1,124 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎虎 - @file： base_variable_splitting_node.py - @date：2025/10/13 15:02 - @desc: -""" -import json -import re - -from django.db.models import QuerySet -from django.utils.translation import gettext_lazy as _ -from langchain_core.messages import HumanMessage -from langchain_core.prompts import PromptTemplate - -from application.flow.i_step_node import NodeResult -from application.flow.step_node.parameter_extraction_node.i_parameter_extraction_node import IParameterExtractionNode -from models_provider.models import Model -from models_provider.tools import get_model_instance_by_model_workspace_id, get_model_credential - -prompt = """ -Please strictly process the text according to the following requirements: -**Task**: -Extract specified field information from given text - -**Enter text**: -{{question}} - -**Extract configuration**: -{{properties}} - -**Rule**: -- Strictly follow the data and field of Extract configuration -- If not found, use null value -- Only return pure JSON without additional text -- Keep the string format neat -""" - - -def get_default_model_params_setting(model_id): - model = QuerySet(Model).filter(id=model_id).first() - credential = get_model_credential(model.provider, model.model_type, model.model_name) - model_params_setting = credential.get_model_params_setting_form( - model.model_name).get_default_form_data() - return model_params_setting - - -def generate_properties(variable_list): - return {variable['field']: {'type': variable['parameter_type'], 'description': (variable.get('desc') or ""), - 'title': variable['label']} for variable in - variable_list} - - -def generate_example(variable_list): - return {variable['field']: None for variable in variable_list} - - -def generate_content(input_variable, variable_list): - properties = generate_properties(variable_list) - prompt_template = PromptTemplate.from_template(prompt, template_format='jinja2') - value = prompt_template.format(properties=properties, question=input_variable) - return value - - -def json_loads(response, variable_list): - if not response or not isinstance(response, str): - return generate_example(variable_list) - - cleaned = response.strip() - - extraction_strategies = [ - lambda: json.loads(cleaned), - lambda: json.loads(re.search(r'```(?:json)?\s*(\{.*?\})\s*```', cleaned, re.DOTALL).group(1)), - lambda: json.loads(re.search(r'(\{.*\})', cleaned, flags=re.DOTALL).group(1)), - ] - for strategy in extraction_strategies: - try: - result = strategy() - return result - except: - continue - return generate_example(variable_list) - - -class BaseParameterExtractionNode(IParameterExtractionNode): - - def save_context(self, details, workflow_manage): - for key, value in details.get('result').items(): - self.context[key] = value - self.context['result'] = details.get('result') - self.context['request'] = details.get('request') - self.context['exception_message'] = details.get('err_message') - - def execute(self, input_variable, variable_list, model_params_setting, model_id, **kwargs) -> NodeResult: - input_variable = str(input_variable) - self.context['request'] = input_variable - - if not model_id: - raise Exception(_('Model is not allowed to be empty')) - - if model_params_setting is None and model_id: - model_params_setting = get_default_model_params_setting(model_id) - workspace_id = self.workflow_manage.get_body().get('workspace_id') - chat_model = get_model_instance_by_model_workspace_id(model_id, workspace_id, - **(model_params_setting or {})) - - content = generate_content(input_variable, variable_list) - response = chat_model.invoke([HumanMessage(content=content)]) - result = json_loads(response.content, variable_list) - return NodeResult({'result': result, **result}, {}) - - def get_details(self, index: int, **kwargs): - return { - 'name': self.node.properties.get('stepName'), - "index": index, - 'run_time': self.context.get('run_time'), - 'type': self.node.type, - 'request': self.context.get('request'), - 'result': self.context.get('result'), - 'status': self.status, - 'err_message': self.err_message, - 'enableException': self.node.properties.get('enableException'), - } diff --git a/apps/application/flow/step_node/question_node/__init__.py b/apps/application/flow/step_node/question_node/__init__.py deleted file mode 100644 index 98a1afcd904..00000000000 --- a/apps/application/flow/step_node/question_node/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： __init__.py - @date：2024/6/11 15:30 - @desc: -""" -from .impl import * diff --git a/apps/application/flow/step_node/question_node/i_question_node.py b/apps/application/flow/step_node/question_node/i_question_node.py deleted file mode 100644 index 2e58b31ea01..00000000000 --- a/apps/application/flow/step_node/question_node/i_question_node.py +++ /dev/null @@ -1,55 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： i_chat_node.py - @date：2024/6/4 13:58 - @desc: -""" -from typing import Type - -from django.utils.translation import gettext_lazy as _ -from rest_framework import serializers - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import INode, NodeResult - - -class QuestionNodeSerializer(serializers.Serializer): - model_id = serializers.CharField(required=False, allow_blank=True, allow_null=True, label=_("Model id")) - model_id_type = serializers.CharField(required=False, default='custom', label=_("Model id type")) - model_id_reference = serializers.ListField(required=False, child=serializers.CharField(), allow_empty=True, - label=_("Reference Field")) - system = serializers.CharField(required=False, allow_blank=True, allow_null=True, - label=_("Role Setting")) - prompt = serializers.CharField(required=True, label=_("Prompt word")) - # 多轮对话数量 - dialogue_number = serializers.IntegerField(required=True, label= - _("Number of multi-round conversations")) - - is_result = serializers.BooleanField(required=False, - label=_('Whether to return content')) - model_params_setting = serializers.DictField(required=False, - label=_("Model parameter settings")) - - -class IQuestionNode(INode): - type = 'question-node' - support = [WorkflowMode.APPLICATION, WorkflowMode.APPLICATION_LOOP, WorkflowMode.KNOWLEDGE, - WorkflowMode.KNOWLEDGE_LOOP, WorkflowMode.TOOL, WorkflowMode.TOOL_LOOP] - - def get_node_params_serializer_class(self) -> Type[serializers.Serializer]: - return QuestionNodeSerializer - - def _run(self): - if [WorkflowMode.KNOWLEDGE, WorkflowMode.KNOWLEDGE_LOOP, WorkflowMode.TOOL, - WorkflowMode.TOOL_LOOP].__contains__(self.workflow_manage.flow.workflow_mode): - return self.execute(**self.node_params_serializer.data, **self.flow_params_serializer.data, - **{'history_chat_record': [], 'stream': True, 'chat_id': None, 'chat_record_id': None}) - else: - return self.execute(**self.node_params_serializer.data, **self.flow_params_serializer.data) - - def execute(self, model_id, system, prompt, dialogue_number, history_chat_record, stream, chat_id, chat_record_id, - model_params_setting=None, model_id_type=None, model_id_reference=None, - **kwargs) -> NodeResult: - pass diff --git a/apps/application/flow/step_node/question_node/impl/__init__.py b/apps/application/flow/step_node/question_node/impl/__init__.py deleted file mode 100644 index d85aa8724ac..00000000000 --- a/apps/application/flow/step_node/question_node/impl/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： __init__.py - @date：2024/6/11 15:35 - @desc: -""" -from .base_question_node import BaseQuestionNode diff --git a/apps/application/flow/step_node/question_node/impl/base_question_node.py b/apps/application/flow/step_node/question_node/impl/base_question_node.py deleted file mode 100644 index 665ba99326d..00000000000 --- a/apps/application/flow/step_node/question_node/impl/base_question_node.py +++ /dev/null @@ -1,173 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： base_question_node.py - @date：2024/6/4 14:30 - @desc: -""" -import re -import time -from functools import reduce -from typing import List, Dict - -from django.db.models import QuerySet -from django.utils.translation import gettext_lazy as _ -from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage - -from application.flow.i_step_node import NodeResult, INode -from application.flow.step_node.question_node.i_question_node import IQuestionNode -from models_provider.models import Model -from models_provider.tools import get_model_instance_by_model_workspace_id, get_model_credential - - -def _write_context(node_variable: Dict, workflow_variable: Dict, node: INode, workflow, answer: str): - chat_model = node_variable.get('chat_model') - message_tokens = chat_model.get_num_tokens_from_messages(node_variable.get('message_list')) - answer_tokens = chat_model.get_num_tokens(answer) - node.context['message_tokens'] = message_tokens - node.context['answer_tokens'] = answer_tokens - node.context['answer'] = answer - node.context['history_message'] = node_variable['history_message'] - node.context['question'] = node_variable['question'] - node.context['run_time'] = time.time() - node.context['start_time'] - if workflow.is_result(node, NodeResult(node_variable, workflow_variable)): - node.answer_text = answer - - -def write_context_stream(node_variable: Dict, workflow_variable: Dict, node: INode, workflow): - """ - 写入上下文数据 (流式) - @param node_variable: 节点数据 - @param workflow_variable: 全局数据 - @param node: 节点 - @param workflow: 工作流管理器 - """ - response = node_variable.get('result') - answer = '' - for chunk in response: - answer += chunk.content - yield chunk.content - _write_context(node_variable, workflow_variable, node, workflow, answer) - - -def write_context(node_variable: Dict, workflow_variable: Dict, node: INode, workflow): - """ - 写入上下文数据 - @param node_variable: 节点数据 - @param workflow_variable: 全局数据 - @param node: 节点实例对象 - @param workflow: 工作流管理器 - """ - response = node_variable.get('result') - answer = response.content - _write_context(node_variable, workflow_variable, node, workflow, answer) - - -def get_default_model_params_setting(model_id): - model = QuerySet(Model).filter(id=model_id).first() - credential = get_model_credential(model.provider, model.model_type, model.model_name) - model_params_setting = credential.get_model_params_setting_form( - model.model_name).get_default_form_data() - return model_params_setting - - -class BaseQuestionNode(IQuestionNode): - def save_context(self, details, workflow_manage): - self.context['run_time'] = details.get('run_time') - self.context['question'] = details.get('question') - self.context['answer'] = details.get('answer') - self.context['message_tokens'] = details.get('message_tokens') - self.context['answer_tokens'] = details.get('answer_tokens') - self.context['exception_message'] = details.get('err_message') - if self.node_params.get('is_result', False): - self.answer_text = details.get('answer') - - def execute(self, model_id, system, prompt, dialogue_number, history_chat_record, stream, chat_id, chat_record_id, - model_params_setting=None, model_id_type=None, model_id_reference=None, - **kwargs) -> NodeResult: - # 处理引用类型 - if model_id_type == 'reference' and model_id_reference: - reference_data = self.workflow_manage.get_reference_field( - model_id_reference[0], - model_id_reference[1:], - ) - if reference_data and isinstance(reference_data, dict): - model_id = reference_data.get('model_id', model_id) - model_params_setting = reference_data.get('model_params_setting') - if not model_id: - raise Exception(_('Model is not allowed to be empty')) - - if model_params_setting is None and model_id: - model_params_setting = get_default_model_params_setting(model_id) - workspace_id = self.workflow_manage.get_body().get('workspace_id') - chat_model = get_model_instance_by_model_workspace_id(model_id, workspace_id, - **(model_params_setting or {})) - history_message = self.get_history_message(history_chat_record, dialogue_number) - self.context['history_message'] = history_message - question = self.generate_prompt_question(prompt) - self.context['question'] = question.content - system = self.workflow_manage.generate_prompt(system) - self.context['system'] = system - message_list = self.generate_message_list(system, prompt, history_message) - self.context['message_list'] = message_list - if stream: - r = chat_model.stream(message_list) - return NodeResult({'result': r, 'chat_model': chat_model, 'message_list': message_list, - 'history_message': history_message, 'question': question.content}, {}, - _write_context=write_context_stream) - else: - r = chat_model.invoke(message_list) - return NodeResult({'result': r, 'chat_model': chat_model, 'message_list': message_list, - 'history_message': history_message, 'question': question.content}, {}, - _write_context=write_context) - - @staticmethod - def get_history_message(history_chat_record, dialogue_number): - start_index = len(history_chat_record) - dialogue_number - history_message = reduce(lambda x, y: [*x, *y], [ - [history_chat_record[index].get_human_message(), history_chat_record[index].get_ai_message()] - for index in - range(start_index if start_index > 0 else 0, len(history_chat_record))], []) - for message in history_message: - if isinstance(message.content, str): - message.content = re.sub(r'.*?', '', message.content, flags=re.DOTALL) - return history_message - - def generate_prompt_question(self, prompt): - return HumanMessage(self.workflow_manage.generate_prompt(prompt)) - - def generate_message_list(self, system: str, prompt: str, history_message): - if system is not None and len(system) > 0: - return [SystemMessage(self.workflow_manage.generate_prompt(system)), *history_message, - HumanMessage(self.workflow_manage.generate_prompt(prompt))] - else: - return [*history_message, HumanMessage(self.workflow_manage.generate_prompt(prompt))] - - @staticmethod - def reset_message_list(message_list: List[BaseMessage], answer_text): - result = [{'role': 'user' if isinstance(message, HumanMessage) else 'ai', 'content': message.content} for - message - in - message_list] - result.append({'role': 'ai', 'content': answer_text}) - return result - - def get_details(self, index: int, **kwargs): - return { - 'name': self.node.properties.get('stepName'), - "index": index, - 'run_time': self.context.get('run_time'), - 'system': self.context.get('system'), - 'history_message': [{'content': message.content, 'role': message.type} for message in - (self.context.get('history_message') if self.context.get( - 'history_message') is not None else [])], - 'question': self.context.get('question'), - 'answer': self.context.get('answer'), - 'type': self.node.type, - 'message_tokens': self.context.get('message_tokens'), - 'answer_tokens': self.context.get('answer_tokens'), - 'status': self.status, - 'err_message': self.err_message, - 'enableException': self.node.properties.get('enableException'), - } diff --git a/apps/application/flow/step_node/reranker_node/__init__.py b/apps/application/flow/step_node/reranker_node/__init__.py deleted file mode 100644 index 881d0f8a393..00000000000 --- a/apps/application/flow/step_node/reranker_node/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎 - @file： __init__.py - @date：2024/9/4 11:37 - @desc: -""" -from .impl import * diff --git a/apps/application/flow/step_node/reranker_node/i_reranker_node.py b/apps/application/flow/step_node/reranker_node/i_reranker_node.py deleted file mode 100644 index af87a6f2003..00000000000 --- a/apps/application/flow/step_node/reranker_node/i_reranker_node.py +++ /dev/null @@ -1,84 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎 - @file： i_reranker_node.py - @date：2024/9/4 10:40 - @desc: -""" -from typing import Type - -from rest_framework import serializers - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import INode, NodeResult - -from django.utils.translation import gettext_lazy as _ - - -class RerankerSettingSerializer(serializers.Serializer): - # 需要查询的条数 - top_n = serializers.IntegerField(required=True, - label=_("Reference segment number")) - # 相似度 0-1之间 - similarity = serializers.FloatField(required=True, max_value=2, min_value=0, - label=_("Reference segment number")) - max_paragraph_char_number = serializers.IntegerField(required=True, - label=_("Maximum number of words in a quoted segment")) - - -class RerankerStepNodeSerializer(serializers.Serializer): - reranker_setting = RerankerSettingSerializer(required=True) - - question_reference_address = serializers.ListField(required=True) - reranker_model_id = serializers.CharField(required=False, allow_blank=True, allow_null=True) - reranker_model_id_type = serializers.CharField(required=False, default='custom') - reranker_model_id_reference = serializers.ListField(required=False, child=serializers.CharField(), allow_empty=True) - reranker_reference_list = serializers.ListField(required=True, child=serializers.ListField(required=True)) - show_knowledge = serializers.BooleanField(required=True, - label=_("The results are displayed in the knowledge sources")) - - def is_valid(self, *, raise_exception=False): - super().is_valid(raise_exception=True) - - -class IRerankerNode(INode): - type = 'reranker-node' - support = [WorkflowMode.APPLICATION, WorkflowMode.APPLICATION_LOOP, WorkflowMode.TOOL, WorkflowMode.TOOL_LOOP] - - def get_node_params_serializer_class(self) -> Type[serializers.Serializer]: - return RerankerStepNodeSerializer - - def _run(self): - question = self.workflow_manage.get_reference_field( - self.node_params_serializer.data.get('question_reference_address')[0], - self.node_params_serializer.data.get('question_reference_address')[1:]) - reranker_list = [self.workflow_manage.get_reference_field( - reference[0], - reference[1:]) for reference in - self.node_params_serializer.data.get('reranker_reference_list')] - - node_params_data = dict(self.node_params_serializer.data) - - reranker_model_id_type = node_params_data.pop('reranker_model_id_type', None) - reranker_model_id_reference = node_params_data.pop('reranker_model_id_reference', None) - reranker_model_id = node_params_data.pop('reranker_model_id', None) - - # 处理引用类型 - if reranker_model_id_type == 'reference' and reranker_model_id_reference: - reference_data = self.workflow_manage.get_reference_field( - reranker_model_id_reference[0], - reranker_model_id_reference[1:], - ) - if reference_data and isinstance(reference_data, dict): - reranker_model_id = reference_data.get('reranker_model_id', - reference_data.get('model_id', reranker_model_id)) - if reranker_model_id is None or reranker_model_id == '': - raise Exception(_('Model is not allowed to be empty')) - - return self.execute(**node_params_data, question=str(question), - reranker_list=reranker_list, reranker_model_id=reranker_model_id) - - def execute(self, question, reranker_setting, reranker_list, reranker_model_id, show_knowledge, - **kwargs) -> NodeResult: - pass diff --git a/apps/application/flow/step_node/reranker_node/impl/__init__.py b/apps/application/flow/step_node/reranker_node/impl/__init__.py deleted file mode 100644 index ef5ca80585b..00000000000 --- a/apps/application/flow/step_node/reranker_node/impl/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎 - @file： __init__.py - @date：2024/9/4 11:39 - @desc: -""" -from .base_reranker_node import * diff --git a/apps/application/flow/step_node/reranker_node/impl/base_reranker_node.py b/apps/application/flow/step_node/reranker_node/impl/base_reranker_node.py deleted file mode 100644 index 36dd2144aee..00000000000 --- a/apps/application/flow/step_node/reranker_node/impl/base_reranker_node.py +++ /dev/null @@ -1,129 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎 - @file： base_reranker_node.py - @date：2024/9/4 11:41 - @desc: -""" -from typing import List - -from langchain_core.documents import Document - -from application.flow.i_step_node import NodeResult -from application.flow.step_node.reranker_node.i_reranker_node import IRerankerNode -from models_provider.tools import get_model_instance_by_model_workspace_id - - -def merge_reranker_list(reranker_list, result=None): - if result is None: - result = [] - for document in reranker_list: - if isinstance(document, list): - merge_reranker_list(document, result) - elif isinstance(document, dict): - content = document.get('title', '') + document.get('content', '') - title = document.get("title") - result.append( - Document(page_content=str(document) if len(content) == 0 else content, - metadata={'title': title, **document})) - else: - result.append(Document(page_content=str(document), metadata={})) - return result - - -def filter_result(document_list: List[Document], max_paragraph_char_number, top_n, similarity): - use_len = 0 - result = [] - for index in range(len(document_list)): - document = document_list[index] - if use_len >= max_paragraph_char_number or index >= top_n or document.metadata.get( - 'relevance_score') < similarity: - break - content = document.page_content[0:max_paragraph_char_number - use_len] - use_len = use_len + len(content) - result.append({'page_content': content, 'metadata': document.metadata}) - return result - - -def reset_result_list(result_list: List[Document], document_list: List[Document]): - r = [] - document_list = document_list.copy() - for result in result_list: - filter_result_list = [document for document in document_list if document.page_content == result.page_content] - if len(filter_result_list) > 0: - item = filter_result_list[0] - document_list.remove(item) - r.append(Document(page_content=item.page_content, - metadata={**item.metadata, 'relevance_score': result.metadata.get('relevance_score')})) - else: - r.append(result) - return r - - -def get_none_result(question): - return NodeResult( - {'document_list': [], 'question': question, - 'result_list': [], 'result': ''}, {}) - - -def reset_metadata(metadata): - meta = metadata.get('meta') - if isinstance(metadata.get('meta'), dict): - if not meta.get('allow_download', False): - metadata['meta'] = {'allow_download': False} - return metadata - - -class BaseRerankerNode(IRerankerNode): - def save_context(self, details, workflow_manage): - self.context['document_list'] = details.get('document_list', []) - self.context['question'] = details.get('question') - self.context['run_time'] = details.get('run_time') - self.context['result_list'] = details.get('result_list') - self.context['result'] = details.get('result') - self.context['exception_message'] = details.get('err_message') - - def execute(self, question, reranker_setting, reranker_list, reranker_model_id, show_knowledge, - **kwargs) -> NodeResult: - self.context['show_knowledge'] = show_knowledge - documents = merge_reranker_list(reranker_list) - documents = [d for d in documents if d.page_content and len(d.page_content) > 0] - if len(documents) == 0: - return get_none_result(question) - top_n = reranker_setting.get('top_n', 3) - self.context['document_list'] = [ - {'page_content': document.page_content, 'metadata': reset_metadata(document.metadata)} for - document in documents] - self.context['question'] = question - workspace_id = self.workflow_manage.get_body().get('workspace_id') - reranker_model = get_model_instance_by_model_workspace_id(reranker_model_id, - workspace_id, - top_n=top_n) - result = reranker_model.compress_documents( - documents, - question) - similarity = reranker_setting.get('similarity', 0.6) - max_paragraph_char_number = reranker_setting.get('max_paragraph_char_number', 5000) - result = reset_result_list(result, documents) - r = filter_result(result, max_paragraph_char_number, top_n, similarity) - return NodeResult({'result_list': r, 'result': ''.join([item.get('page_content') for item in r]), - 'is_hit_handling_method_list': [r for row in r if - row.get('metadata').get('is_hit_handling_method')]}, {}) - - def get_details(self, index: int, **kwargs): - return { - 'show_knowledge': self.context.get('show_knowledge'), - 'name': self.node.properties.get('stepName'), - "index": index, - 'document_list': self.context.get('document_list'), - "question": self.context.get('question'), - 'run_time': self.context.get('run_time'), - 'type': self.node.type, - 'reranker_setting': self.node_params_serializer.data.get('reranker_setting'), - 'result_list': self.context.get('result_list'), - 'result': self.context.get('result'), - 'status': self.status, - 'err_message': self.err_message, - 'enableException': self.node.properties.get('enableException'), - } diff --git a/apps/application/flow/step_node/search_document_node/__init__.py b/apps/application/flow/step_node/search_document_node/__init__.py deleted file mode 100644 index ce8f10f3e24..00000000000 --- a/apps/application/flow/step_node/search_document_node/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .impl import * \ No newline at end of file diff --git a/apps/application/flow/step_node/search_document_node/i_search_document_node.py b/apps/application/flow/step_node/search_document_node/i_search_document_node.py deleted file mode 100644 index 0a2c99a1e71..00000000000 --- a/apps/application/flow/step_node/search_document_node/i_search_document_node.py +++ /dev/null @@ -1,58 +0,0 @@ -# coding=utf-8 -from typing import Type, List - -from django.utils.translation import gettext_lazy as _ -from rest_framework import serializers - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import INode, NodeResult - - -class SearchDocumentStepNodeSerializer(serializers.Serializer): - knowledge_id_list = serializers.ListField( - required=False, child=serializers.UUIDField(required=True), - label=_("knowledge id list"), default=list - ) - search_mode = serializers.ChoiceField( - required=False, choices=['auto', 'custom'], label=_("search mode"), default='auto' - ) - search_scope_type = serializers.ChoiceField( - required=False, choices=['custom', 'referencing'], label=_("search scope type"), - allow_null=True, default='custom' - ) - search_scope_source = serializers.ChoiceField( - required=False, choices=['document', 'knowledge'], - label=_("search scope variable type"), default='knowledge' - ) - search_scope_reference = serializers.ListField( - required=False, label=_("search scope variable"), default=list - ) - question_reference = serializers.ListField( - required=False, label=_("question reference address"), default=list - ) - search_condition_type = serializers.ChoiceField( - required=False, choices=['AND', 'OR'], label=_("search condition type"), default='AND' - ) - search_condition_list = serializers.ListField( - required=False, label=_("search condition list"), default=list - ) - - def is_valid(self, *, raise_exception=False): - super().is_valid(raise_exception=True) - - -class ISearchDocumentStepNode(INode): - type = 'search-document-node' - support = [WorkflowMode.APPLICATION, WorkflowMode.APPLICATION_LOOP, WorkflowMode.TOOL, WorkflowMode.TOOL_LOOP] - - def get_node_params_serializer_class(self) -> Type[serializers.Serializer]: - return SearchDocumentStepNodeSerializer - - def _run(self): - return self.execute(**self.node_params_serializer.data, **self.flow_params_serializer.data) - - def execute(self, knowledge_id_list: List, search_mode: str, search_scope_type: str, search_scope_source: str, - search_scope_reference: List, question_reference: List, search_condition_type: str, - search_condition_list: List, - **kwargs) -> NodeResult: - pass diff --git a/apps/application/flow/step_node/search_document_node/impl/__init__.py b/apps/application/flow/step_node/search_document_node/impl/__init__.py deleted file mode 100644 index 74a1aa384a7..00000000000 --- a/apps/application/flow/step_node/search_document_node/impl/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .base_search_document_node import BaseSearchDocumentNode diff --git a/apps/application/flow/step_node/search_document_node/impl/base_search_document_node.py b/apps/application/flow/step_node/search_document_node/impl/base_search_document_node.py deleted file mode 100644 index d118baa9424..00000000000 --- a/apps/application/flow/step_node/search_document_node/impl/base_search_document_node.py +++ /dev/null @@ -1,212 +0,0 @@ -# coding=utf-8 -from typing import List - -import jieba -from django.db.models import Q -from django.db.models import QuerySet - -from application.flow.i_step_node import NodeResult -from application.flow.step_node.search_document_node.i_search_document_node import ISearchDocumentStepNode -from common.auth.constants.role_constants import RoleConstants -from common.database_model_manage.database_model_manage import DatabaseModelManage -from common.utils.shared_resource_auth import filter_authorized_ids -from knowledge.models import Document, DocumentTag, Knowledge - - -class BaseSearchDocumentNode(ISearchDocumentStepNode): - def save_context(self, details, workflow_manage): - self.context['document_list'] = details.get('document_list') - self.context['knowledge_list'] = details.get('knowledge_list') - self.context['document_items'] = details.get('document_items') - self.context['knowledge_items'] = details.get('knowledge_items') - self.context['question'] = details.get('question') - self.context['run_time'] = details.get('run_time') - self.context['exception_message'] = details.get('err_message') - - def get_reference_content(self, fields: List[str]): - return self.workflow_manage.get_reference_field(fields[0], fields[1:]) if fields else None - - def execute(self, knowledge_id_list: List, search_mode: str, search_scope_type: str, search_scope_source: str, - search_scope_reference: List, question_reference: List, search_condition_type: str, - search_condition_list: List, - **kwargs) -> NodeResult: - workspace_id = self.workflow_manage.get_body().get('workspace_id') - - if search_scope_type == 'custom': # 手动选择知识库 - knowledge_id_list = filter_authorized_ids('knowledge', knowledge_id_list, workspace_id) - document_id_list = QuerySet(Document).filter( - knowledge_id__in=knowledge_id_list - ).values_list('id', flat=True) - else: # 引用上一步知识库/文档 - if search_scope_source == 'document': # 文档 - document_id_list = self.get_reference_content(search_scope_reference) - else: # 知识库 - ref_knowledge_ids = filter_authorized_ids('knowledge', - self.get_reference_content(search_scope_reference), - workspace_id) - document_id_list = QuerySet(Document).filter( - knowledge_id__in=ref_knowledge_ids - ).values_list('id', flat=True) - - # 权限过滤 - get_knowledge_list_of_authorized = DatabaseModelManage.get_model('get_knowledge_list_of_authorized') - chat_user_type = self.workflow_manage.get_body().get('chat_user_type') - - if get_knowledge_list_of_authorized is not None and RoleConstants.CHAT_USER.value.name == chat_user_type: - actual_knowledge_ids = list( - QuerySet(Document).filter(id__in=document_id_list) - .values_list('knowledge_id', flat=True).distinct() - ) - authorized_knowledge_ids = get_knowledge_list_of_authorized( - self.workflow_manage.get_body().get('chat_user_id'), - [str(k_id) for k_id in actual_knowledge_ids] - ) - document_id_list = QuerySet(Document).filter( - id__in=document_id_list, - knowledge_id__in=authorized_knowledge_ids - ).values_list('id', flat=True) - - if search_mode == 'auto': # 通过问题自动检索 - matched_doc_ids = self.handle_auto_tags(document_id_list, question_reference) - - final_document_ids = list(matched_doc_ids) - else: # 自定义检索条件 - matched_document_ids = self.handle_custom_tags( - document_id_list, search_condition_list, search_condition_type - ) - - final_document_ids = list(matched_document_ids) - - # UUID to str - final_document_ids = [str(doc_id) for doc_id in final_document_ids] - document_items = QuerySet(Document).filter(id__in=final_document_ids).values() - final_knowledge_ids = list(set(str(doc['knowledge_id']) for doc in document_items)) - knowledge_items = QuerySet(Knowledge).filter(id__in=final_knowledge_ids).values() - - return NodeResult({ - 'document_list': final_document_ids, - 'document_items': list(document_items), - 'knowledge_list': final_knowledge_ids, - 'knowledge_items': list(knowledge_items) - }, {}) - - def handle_auto_tags(self, document_id_list: list, question_reference: list): - question = self.get_reference_content(question_reference) - - # 使用jieba分词 - keywords = jieba.lcut(question) - if not keywords: - return set() - - # 构建OR查询,一次性获取所有匹配的文档 - q_objects = Q() - for keyword in keywords: - q_objects |= Q(tag__value__icontains=keyword) - - # 单次数据库查询 - matched_doc_ids = set( - QuerySet(DocumentTag) - .filter(document_id__in=document_id_list) - .filter(q_objects) - .values_list('document_id', flat=True) - .distinct() - ) - - return matched_doc_ids - - def handle_custom_tags(self, document_id_list: List, search_condition_list: list, search_condition_type: str): - - if not search_condition_list: - return set(document_id_list) - - if search_condition_type == 'AND': - # AND逻辑:使用子查询和聚合 - matched_doc_ids = set(document_id_list) - - for condition in search_condition_list: - tag_key = condition['key'] - field_value = self.workflow_manage.generate_prompt(condition['value']) - compare_type = condition['compare'] - - if not field_value or field_value == 'None' or len(field_value) == 0: - continue - - # 构建查询条件 - if compare_type == 'not_contain': - # 反向查询:找出包含该标签的文档,然后排除 - exclude_docs = set(QuerySet(DocumentTag).filter( - document_id__in=matched_doc_ids, - tag__key=tag_key, - tag__value__icontains=field_value - ).values_list('document_id', flat=True).distinct()) - - matched_doc_ids = matched_doc_ids - exclude_docs - else: - if compare_type == 'contain': - q_filter = Q(tag__key=tag_key, tag__value__icontains=field_value) - elif compare_type == 'eq': - q_filter = Q(tag__key=tag_key, tag__value=field_value) - else: - continue - - # 单次查询获取符合条件的文档 - tag_docs = set(QuerySet(DocumentTag).filter( - document_id__in=matched_doc_ids - ).filter(q_filter).values_list('document_id', flat=True).distinct()) - - matched_doc_ids = matched_doc_ids.intersection(tag_docs) - - return matched_doc_ids - - else: - # OR逻辑 - matched_docs = set() - - for condition in search_condition_list: - tag_key = condition['key'] - field_value = self.workflow_manage.generate_prompt(condition['value']) - compare_type = condition['compare'] - - if not field_value or field_value == 'None' or len(field_value) == 0: - continue - - if compare_type == 'not_contain': - # 反向查询:找出包含该标签的文档,然后用全集减去 - exclude_docs = set(QuerySet(DocumentTag).filter( - document_id__in=document_id_list, - tag__key=tag_key, - tag__value__icontains=field_value - ).values_list('document_id', flat=True).distinct()) - - matched_docs = matched_docs.union(set(document_id_list) - exclude_docs) - else: - if compare_type == 'contain': - q_filter = Q(tag__key=tag_key, tag__value__icontains=field_value) - elif compare_type == 'eq': - q_filter = Q(tag__key=tag_key, tag__value=field_value) - else: - continue - - docs = set(QuerySet(DocumentTag).filter( - document_id__in=document_id_list - ).filter(q_filter).values_list('document_id', flat=True).distinct()) - - matched_docs = matched_docs.union(docs) - - return matched_docs - - def get_details(self, index: int, **kwargs): - return { - 'name': self.node.properties.get('stepName'), - 'question': self.context.get('question'), - "index": index, - 'run_time': self.context.get('run_time'), - 'document_list': self.context.get('document_list'), - 'knowledge_list': self.context.get('knowledge_list'), - 'document_items': self.context.get('document_items'), - 'knowledge_items': self.context.get('knowledge_items'), - 'type': self.node.type, - 'status': self.status, - 'err_message': self.err_message, - 'enableException': self.node.properties.get('enableException'), - } diff --git a/apps/application/flow/step_node/search_knowledge_node/__init__.py b/apps/application/flow/step_node/search_knowledge_node/__init__.py deleted file mode 100644 index 98a1afcd904..00000000000 --- a/apps/application/flow/step_node/search_knowledge_node/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： __init__.py - @date：2024/6/11 15:30 - @desc: -""" -from .impl import * diff --git a/apps/application/flow/step_node/search_knowledge_node/i_search_knowledge_node.py b/apps/application/flow/step_node/search_knowledge_node/i_search_knowledge_node.py deleted file mode 100644 index 0cf23cb5e5d..00000000000 --- a/apps/application/flow/step_node/search_knowledge_node/i_search_knowledge_node.py +++ /dev/null @@ -1,96 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： i_search_dataset_node.py - @date：2024/6/3 17:52 - @desc: -""" -import re -from typing import Type - -from django.core import validators -from django.utils.translation import gettext_lazy as _ -from rest_framework import serializers - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import INode, NodeResult -from common.utils.common import flat_map - - -class DatasetSettingSerializer(serializers.Serializer): - # 需要查询的条数 - top_n = serializers.IntegerField(required=True, - label=_("Reference segment number")) - # 相似度 0-1之间 - similarity = serializers.FloatField(required=True, max_value=2, min_value=0, - label=_('similarity')) - search_mode = serializers.CharField(required=True, validators=[ - validators.RegexValidator(regex=re.compile("^embedding|keywords|blend$"), - message=_("The type only supports embedding|keywords|blend"), code=500) - ], label=_("Retrieval Mode")) - max_paragraph_char_number = serializers.IntegerField(required=True, - label=_("Maximum number of words in a quoted segment")) - - -class SearchDatasetStepNodeSerializer(serializers.Serializer): - # 需要查询的数据集id列表 - knowledge_id_list = serializers.ListField(required=True, child=serializers.UUIDField(required=True), - label=_("Dataset id list")) - knowledge_setting = DatasetSettingSerializer(required=True) - - question_reference_address = serializers.ListField(required=True) - - show_knowledge = serializers.BooleanField(required=True, - label=_("The results are displayed in the knowledge sources")) - search_scope_type = serializers.ChoiceField( - required=False, choices=['custom', 'referencing'], label=_("search scope type"), - allow_null=True, default='custom' - ) - search_scope_source = serializers.ChoiceField( - required=False, choices=['document', 'knowledge'], - label=_("search scope variable type"), default='knowledge' - ) - search_scope_reference = serializers.ListField( - required=False, label=_("search scope variable"), default=list - ) - - def is_valid(self, *, raise_exception=False): - super().is_valid(raise_exception=True) - - -def get_paragraph_list(chat_record, node_id): - return flat_map([chat_record.details[key].get('paragraph_list', []) for key in chat_record.details if - (chat_record.details[ - key].get('type', '') == 'search-dataset-node') and chat_record.details[key].get( - 'paragraph_list', []) is not None and key == node_id]) - - -class ISearchKnowledgeStepNode(INode): - type = 'search-knowledge-node' - support = [WorkflowMode.APPLICATION, WorkflowMode.APPLICATION_LOOP, WorkflowMode.TOOL, WorkflowMode.TOOL_LOOP] - - def get_node_params_serializer_class(self) -> Type[serializers.Serializer]: - return SearchDatasetStepNodeSerializer - - def _run(self): - question = self.workflow_manage.get_reference_field( - self.node_params_serializer.data.get('question_reference_address')[0], - self.node_params_serializer.data.get('question_reference_address')[1:]) - exclude_paragraph_id_list = [] - if self.flow_params_serializer.data.get('re_chat', False): - history_chat_record = self.flow_params_serializer.data.get('history_chat_record', []) - paragraph_id_list = [p.get('id') for p in flat_map( - [get_paragraph_list(chat_record, self.runtime_node_id) for chat_record in history_chat_record if - chat_record.problem_text == question])] - exclude_paragraph_id_list = list(set(paragraph_id_list)) - - return self.execute(**self.node_params_serializer.data, question=str(question), - exclude_paragraph_id_list=exclude_paragraph_id_list) - - def execute(self, dataset_id_list, dataset_setting, question, show_knowledge, search_scope_type, - search_scope_source, - search_scope_reference, - exclude_paragraph_id_list=None, - **kwargs) -> NodeResult: - pass diff --git a/apps/application/flow/step_node/search_knowledge_node/impl/__init__.py b/apps/application/flow/step_node/search_knowledge_node/impl/__init__.py deleted file mode 100644 index 76a70567714..00000000000 --- a/apps/application/flow/step_node/search_knowledge_node/impl/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： __init__.py - @date：2024/6/11 15:35 - @desc: -""" -from .base_search_knowledge_node import BaseSearchKnowledgeNode diff --git a/apps/application/flow/step_node/search_knowledge_node/impl/base_search_knowledge_node.py b/apps/application/flow/step_node/search_knowledge_node/impl/base_search_knowledge_node.py deleted file mode 100644 index b11cc950542..00000000000 --- a/apps/application/flow/step_node/search_knowledge_node/impl/base_search_knowledge_node.py +++ /dev/null @@ -1,226 +0,0 @@ -# coding=utf-8 -""" -@project: maxkb -@Author：虎 -@file： base_search_dataset_node.py -@date：2024/6/4 11:56 -@desc: -""" - -import os -from typing import List, Dict - -from django.db import connection -from django.db.models import QuerySet - -from application.flow.i_step_node import NodeResult -from application.flow.step_node.search_knowledge_node.i_search_knowledge_node import ISearchKnowledgeStepNode -from common.config.embedding_config import VectorStore -from common.auth.constants.role_constants import RoleConstants -from common.database_model_manage.database_model_manage import DatabaseModelManage -from common.db.search import native_search -from common.utils.common import get_file_content -from common.utils.shared_resource_auth import filter_authorized_ids -from knowledge.models import Document, Paragraph, Knowledge, SearchMode -from knowledge.services.retrieval_stats import get_recall_tracker, record_recall_safely -from maxkb.conf import PROJECT_DIR -from models_provider.tools import get_model_instance_by_model_workspace_id - - -def get_embedding_id(dataset_id_list): - dataset_list = QuerySet(Knowledge).filter(id__in=dataset_id_list) - if len(set([dataset.embedding_model_id for dataset in dataset_list])) > 1: - raise Exception("关联知识库的向量模型不一致，无法召回分段。") - if len(dataset_list) == 0: - raise Exception("知识库设置错误,请重新设置知识库") - return dataset_list[0].embedding_model_id - - -def get_none_result(question): - return NodeResult( - {"paragraph_list": [], "is_hit_handling_method": [], "question": question, "data": "", "directly_return": ""}, - {}, - ) - - -def reset_title(title): - if title is None or len(title.strip()) == 0: - return "" - else: - return f"#### {title}\n" - - -def reset_meta(meta): - if not meta.get("allow_download", False): - return {"allow_download": False} - return meta - - -class BaseSearchKnowledgeNode(ISearchKnowledgeStepNode): - def save_context(self, details, workflow_manage): - result = details.get("paragraph_list", []) - knowledge_setting = self.node_params_serializer.data.get("knowledge_setting") - directly_return = "\n".join( - [ - f"{paragraph.get('title', '')}:{paragraph.get('content')}" - for paragraph in result - if paragraph.get("is_hit_handling_method") - ] - ) - self.context["paragraph_list"] = result - self.context["question"] = details.get("question") - self.context["run_time"] = details.get("run_time") - self.context["is_hit_handling_method_list"] = [row for row in result if row.get("is_hit_handling_method")] - self.context["data"] = "\n".join( - [f"{paragraph.get('title', '')}:{paragraph.get('content')}" for paragraph in result] - )[0 : knowledge_setting.get("max_paragraph_char_number", 5000)] - self.context["directly_return"] = directly_return - self.context["exception_message"] = details.get("err_message") - - def get_reference_content(self, fields: List[str]): - return self.workflow_manage.get_reference_field(fields[0], fields[1:]) if fields else None - - def execute( - self, - knowledge_id_list, - knowledge_setting, - question, - show_knowledge, - search_scope_type, - search_scope_source, - search_scope_reference, - exclude_paragraph_id_list=None, - **kwargs, - ) -> NodeResult: - self.context["question"] = question - self.context["show_knowledge"] = show_knowledge - - document_id_list = None - if search_scope_type == "referencing": # 引用上一步知识库/文档 - if search_scope_source == "knowledge": # 知识库 - knowledge_id_list = self.get_reference_content(search_scope_reference) - else: # 文档 - document_id_list = self.get_reference_content(search_scope_reference) - knowledge_id_list = [ - str(k) - for k in QuerySet(Document) - .filter(id__in=document_id_list) - .values_list("knowledge_id", flat=True) - .distinct() - ] - - get_knowledge_list_of_authorized = DatabaseModelManage.get_model("get_knowledge_list_of_authorized") - chat_user_type = self.workflow_manage.get_body().get("chat_user_type") - if get_knowledge_list_of_authorized is not None and RoleConstants.CHAT_USER.value.name == chat_user_type: - knowledge_id_list = get_knowledge_list_of_authorized( - self.workflow_manage.get_body().get("chat_user_id"), knowledge_id_list - ) - workspace_id = self.workflow_manage.get_body().get("workspace_id") - knowledge_id_list = filter_authorized_ids("knowledge", knowledge_id_list, workspace_id) - if len(knowledge_id_list) == 0: - return get_none_result(question) - model_id = get_embedding_id(knowledge_id_list) - embedding_model = get_model_instance_by_model_workspace_id(model_id, workspace_id) - embedding_value = embedding_model.embed_query(question) - vector = VectorStore.get_embedding_vector() - exclude_document_id_list = [ - str(document.id) - for document in QuerySet(Document).filter(knowledge_id__in=knowledge_id_list, is_active=False) - ] - embedding_list = vector.query( - question, - embedding_value, - knowledge_id_list, - document_id_list, - exclude_document_id_list, - exclude_paragraph_id_list, - True, - knowledge_setting.get("top_n"), - knowledge_setting.get("similarity"), - SearchMode(knowledge_setting.get("search_mode")), - ) - # 手动关闭数据库连接 - connection.close() - if embedding_list is None: - return get_none_result(question) - paragraph_list = self.list_paragraph(embedding_list, vector) - result = [self.reset_paragraph(paragraph, embedding_list) for paragraph in paragraph_list] - result = sorted(result, key=lambda p: p.get("similarity"), reverse=True) - if not self.workflow_manage.get_body().get("debug", False): - recalled_paragraph_ids = {paragraph.get("id") for paragraph in result} - record_recall_safely( - [ - embedding - for embedding in embedding_list - if str(embedding.get("paragraph_id")) in recalled_paragraph_ids - ], - tracker=get_recall_tracker(self.workflow_manage), - ) - return NodeResult( - { - "paragraph_list": result, - "is_hit_handling_method_list": [row for row in result if row.get("is_hit_handling_method")], - "data": "\n".join( - [f"{reset_title(paragraph.get('title', ''))}{paragraph.get('content')}" for paragraph in result] - )[0 : knowledge_setting.get("max_paragraph_char_number", 5000)], - "directly_return": "\n".join( - [paragraph.get("content") for paragraph in result if paragraph.get("is_hit_handling_method")] - ), - "question": question, - }, - {}, - ) - - @staticmethod - def reset_paragraph(paragraph: Dict, embedding_list: List): - filter_embedding_list = [ - embedding for embedding in embedding_list if str(embedding.get("paragraph_id")) == str(paragraph.get("id")) - ] - if filter_embedding_list is not None and len(filter_embedding_list) > 0: - find_embedding = filter_embedding_list[-1] - return { - **paragraph, - "similarity": find_embedding.get("similarity"), - "is_hit_handling_method": find_embedding.get("similarity") > paragraph.get("directly_return_similarity") - and paragraph.get("hit_handling_method") == "directly_return", - "update_time": paragraph.get("update_time").strftime("%Y-%m-%d %H:%M:%S"), - "create_time": paragraph.get("create_time").strftime("%Y-%m-%d %H:%M:%S"), - "id": str(paragraph.get("id")), - "knowledge_id": str(paragraph.get("knowledge_id")), - "document_id": str(paragraph.get("document_id")), - "meta": reset_meta(paragraph.get("meta")), - } - - @staticmethod - def list_paragraph(embedding_list: List, vector): - paragraph_id_list = [row.get("paragraph_id") for row in embedding_list] - if paragraph_id_list is None or len(paragraph_id_list) == 0: - return [] - paragraph_list = native_search( - QuerySet(Paragraph).filter(id__in=paragraph_id_list), - get_file_content( - os.path.join(PROJECT_DIR, "apps", "application", "sql", "list_knowledge_paragraph_by_paragraph_id.sql") - ), - with_table_name=True, - ) - # 如果向量库中存在脏数据 直接删除 - if len(paragraph_list) != len(paragraph_id_list): - exist_paragraph_list = [row.get("id") for row in paragraph_list] - for paragraph_id in paragraph_id_list: - if not exist_paragraph_list.__contains__(paragraph_id): - vector.delete_by_paragraph_id(paragraph_id) - return paragraph_list - - def get_details(self, index: int, **kwargs): - return { - "name": self.node.properties.get("stepName"), - "show_knowledge": self.context.get("show_knowledge"), - "question": self.context.get("question"), - "index": index, - "run_time": self.context.get("run_time"), - "paragraph_list": self.context.get("paragraph_list"), - "type": self.node.type, - "status": self.status, - "err_message": self.err_message, - "enableException": self.node.properties.get("enableException"), - } diff --git a/apps/application/flow/step_node/speech_to_text_step_node/__init__.py b/apps/application/flow/step_node/speech_to_text_step_node/__init__.py deleted file mode 100644 index f3feecc9ce2..00000000000 --- a/apps/application/flow/step_node/speech_to_text_step_node/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# coding=utf-8 - -from .impl import * diff --git a/apps/application/flow/step_node/speech_to_text_step_node/i_speech_to_text_node.py b/apps/application/flow/step_node/speech_to_text_step_node/i_speech_to_text_node.py deleted file mode 100644 index 32e1bb752fd..00000000000 --- a/apps/application/flow/step_node/speech_to_text_step_node/i_speech_to_text_node.py +++ /dev/null @@ -1,47 +0,0 @@ -# coding=utf-8 - -from typing import Type - -from django.utils.translation import gettext_lazy as _ -from rest_framework import serializers - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import INode, NodeResult - - -class SpeechToTextNodeSerializer(serializers.Serializer): - stt_model_id = serializers.CharField(required=False, allow_blank=True, allow_null=True, label=_("Model id")) - stt_model_id_type = serializers.CharField(required=False, default='custom', label=_("Model id type")) - stt_model_id_reference = serializers.ListField(required=False, child=serializers.CharField(), allow_empty=True, - label=_("Reference Field")) - is_result = serializers.BooleanField(required=False, - label=_('Whether to return content')) - - audio_list = serializers.ListField(required=True, - label=_("The audio file cannot be empty")) - model_params_setting = serializers.DictField(required=False, - label=_("Model parameter settings")) - - -class ISpeechToTextNode(INode): - type = 'speech-to-text-node' - support = [WorkflowMode.APPLICATION, WorkflowMode.APPLICATION_LOOP, WorkflowMode.KNOWLEDGE, - WorkflowMode.KNOWLEDGE_LOOP,WorkflowMode.TOOL,WorkflowMode.TOOL_LOOP] - - def get_node_params_serializer_class(self) -> Type[serializers.Serializer]: - return SpeechToTextNodeSerializer - - def _run(self): - res = self.workflow_manage.get_reference_field(self.node_params_serializer.data.get('audio_list')[0], - self.node_params_serializer.data.get('audio_list')[1:]) - for audio in res: - if 'file_id' not in audio: - raise ValueError( - _("Parameter value error: The uploaded audio lacks file_id, and the audio upload fails")) - - return self.execute(audio=res, **self.node_params_serializer.data, **self.flow_params_serializer.data) - - def execute(self, stt_model_id, - audio, model_params_setting=None, stt_model_id_type=None, stt_model_id_reference=None, - **kwargs) -> NodeResult: - pass diff --git a/apps/application/flow/step_node/speech_to_text_step_node/impl/__init__.py b/apps/application/flow/step_node/speech_to_text_step_node/impl/__init__.py deleted file mode 100644 index 9d2da615820..00000000000 --- a/apps/application/flow/step_node/speech_to_text_step_node/impl/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# coding=utf-8 - -from .base_speech_to_text_node import BaseSpeechToTextNode diff --git a/apps/application/flow/step_node/speech_to_text_step_node/impl/base_speech_to_text_node.py b/apps/application/flow/step_node/speech_to_text_step_node/impl/base_speech_to_text_node.py deleted file mode 100644 index 1df3f85cdeb..00000000000 --- a/apps/application/flow/step_node/speech_to_text_step_node/impl/base_speech_to_text_node.py +++ /dev/null @@ -1,89 +0,0 @@ -# coding=utf-8 -import os -import tempfile -from concurrent.futures import ThreadPoolExecutor - -from django.db.models import QuerySet - -from application.flow.i_step_node import NodeResult -from application.flow.step_node.speech_to_text_step_node.i_speech_to_text_node import ISpeechToTextNode -from common.utils.common import split_and_transcribe, any_to_mp3 -from knowledge.models import File -from models_provider.tools import get_model_instance_by_model_workspace_id - - -class BaseSpeechToTextNode(ISpeechToTextNode): - - def save_context(self, details, workflow_manage): - self.context['answer'] = details.get('answer') - self.context['result'] = details.get('answer') - if self.node_params.get('is_result', False): - self.answer_text = details.get('answer') - self.context['exception_message'] = details.get('err_message') - - def execute(self, stt_model_id, audio, model_params_setting=None, stt_model_id_type=None, stt_model_id_reference=None,**kwargs) -> NodeResult: - - # 处理引用类型 - if stt_model_id_type == 'reference' and stt_model_id_reference: - reference_data = self.workflow_manage.get_reference_field( - stt_model_id_reference[0], - stt_model_id_reference[1:], - ) - if reference_data and isinstance(reference_data, dict): - stt_model_id = reference_data.get('stt_model_id', reference_data.get('model_id', stt_model_id)) - model_params_setting = reference_data.get('model_params_setting') - - from django.utils.translation import gettext_lazy as _ - - if stt_model_id is None or stt_model_id == '': - raise Exception(_('Model is not allowed to be empty')) - workspace_id = self.workflow_manage.get_body().get('workspace_id') - stt_model = get_model_instance_by_model_workspace_id(stt_model_id, workspace_id, **(model_params_setting or {})) - audio_list = audio - self.context['audio_list'] = audio - - def process_audio_item(audio_item, model): - file = QuerySet(File).filter(id=audio_item['file_id']).first() - # 根据file_name 吧文件转成mp3格式 - file_format = file.file_name.split('.')[-1] - with tempfile.NamedTemporaryFile(delete=False, suffix=f'.{file_format}') as temp_file: - temp_file.write(file.get_bytes()) - temp_file_path = temp_file.name - with tempfile.NamedTemporaryFile(delete=False, suffix='.mp3') as temp_amr_file: - temp_mp3_path = temp_amr_file.name - any_to_mp3(temp_file_path, temp_mp3_path) - try: - transcription = split_and_transcribe(temp_mp3_path, model) - return {file.file_name: transcription} - finally: - os.remove(temp_file_path) - os.remove(temp_mp3_path) - - def process_audio_items(audio_list, model): - with ThreadPoolExecutor(max_workers=5) as executor: - results = list(executor.map(lambda item: process_audio_item(item, model), audio_list)) - return results - - result = process_audio_items(audio_list, stt_model) - content = [] - result_content = [] - for item in result: - for key, value in item.items(): - content.append(f'### {key}\n{value}') - result_content.append(value) - return NodeResult({'answer': '\n'.join(result_content), 'result': '\n'.join(result_content), - 'content': content}, {}) - - def get_details(self, index: int, **kwargs): - return { - 'name': self.node.properties.get('stepName'), - "index": index, - 'run_time': self.context.get('run_time'), - 'answer': self.context.get('answer'), - 'content': self.context.get('content'), - 'type': self.node.type, - 'status': self.status, - 'err_message': self.err_message, - 'audio_list': self.context.get('audio_list'), - 'enableException': self.node.properties.get('enableException'), - } diff --git a/apps/application/flow/step_node/start_node/__init__.py b/apps/application/flow/step_node/start_node/__init__.py deleted file mode 100644 index 98a1afcd904..00000000000 --- a/apps/application/flow/step_node/start_node/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： __init__.py - @date：2024/6/11 15:30 - @desc: -""" -from .impl import * diff --git a/apps/application/flow/step_node/start_node/i_start_node.py b/apps/application/flow/step_node/start_node/i_start_node.py deleted file mode 100644 index 40caf0199bf..00000000000 --- a/apps/application/flow/step_node/start_node/i_start_node.py +++ /dev/null @@ -1,21 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： i_start_node.py - @date：2024/6/3 16:54 - @desc: -""" -from application.flow.common import WorkflowMode -from application.flow.i_step_node import INode, NodeResult - - -class IStarNode(INode): - type = 'start-node' - support = [WorkflowMode.APPLICATION] - - def _run(self): - return self.execute(**self.flow_params_serializer.data) - - def execute(self, question, **kwargs) -> NodeResult: - pass diff --git a/apps/application/flow/step_node/start_node/impl/__init__.py b/apps/application/flow/step_node/start_node/impl/__init__.py deleted file mode 100644 index b68a92d021f..00000000000 --- a/apps/application/flow/step_node/start_node/impl/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： __init__.py - @date：2024/6/11 15:36 - @desc: -""" -from .base_start_node import BaseStartStepNode diff --git a/apps/application/flow/step_node/start_node/impl/base_start_node.py b/apps/application/flow/step_node/start_node/impl/base_start_node.py deleted file mode 100644 index 81a23eb25e4..00000000000 --- a/apps/application/flow/step_node/start_node/impl/base_start_node.py +++ /dev/null @@ -1,121 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： base_start_node.py - @date：2024/6/3 17:17 - @desc: -""" -import time -from datetime import datetime -from typing import List, Type - -from django.db.models import QuerySet -from django.utils import timezone -from rest_framework import serializers - -from application.flow.i_step_node import NodeResult -from application.flow.step_node.start_node.i_start_node import IStarNode -from application.models import ApplicationLongTermMemory - - -def get_default_global_variable(input_field_list: List): - return { - item.get('variable') or item.get('field'): item.get('default_value') - for item in input_field_list - if item.get('default_value', None) is not None - } - - -def get_global_variable(node): - body = node.workflow_manage.get_body() - history_chat_record = node.flow_params_serializer.data.get('history_chat_record', []) - history_context = [{'question': chat_record.problem_text, 'answer': chat_record.answer_text} for chat_record in - history_chat_record] - chat_id = node.flow_params_serializer.data.get('chat_id') - return {'time': timezone.localtime(timezone.now()).strftime('%Y-%m-%d %H:%M:%S'), 'start_time': time.time(), - 'history_context': history_context, 'chat_id': str(chat_id), **node.workflow_manage.form_data, - 'chat_user_id': body.get('chat_user_id'), - 'chat_user_type': body.get('chat_user_type'), - 'chat_user': body.get('chat_user'), - 'chat_user_group': body.get('chat_user_group') - } - - -class BaseStartStepNode(IStarNode): - def save_context(self, details, workflow_manage): - base_node = self.workflow_manage.get_base_node() - default_global_variable = get_default_global_variable(base_node.properties.get('user_input_field_list', [])) - default_api_global_variable = get_default_global_variable(base_node.properties.get('api_input_field_list', [])) - workflow_variable = {**default_global_variable, **default_api_global_variable, **get_global_variable(self)} - self.context['question'] = details.get('question') - self.context['run_time'] = details.get('run_time') - self.context['document'] = details.get('document_list') - self.context['image'] = details.get('image_list') - self.context['audio'] = details.get('audio_list') - self.context['video'] = details.get('video_list') - self.context['other'] = details.get('other_list') - self.context['exception_message'] = details.get('err_message') - self.status = details.get('status') - self.err_message = details.get('err_message') - for key, value in workflow_variable.items(): - workflow_manage.context[key] = value - for item in details.get('global_fields', []): - workflow_manage.context[item.get('key')] = item.get('value') - self.workflow_manage.chat_context = self.workflow_manage.get_chat_info().get_chat_variable() - - def get_node_params_serializer_class(self) -> Type[serializers.Serializer]: - pass - - def execute(self, question, **kwargs) -> NodeResult: - base_node = self.workflow_manage.get_base_node() - default_global_variable = get_default_global_variable(base_node.properties.get('user_input_field_list', [])) - default_api_global_variable = get_default_global_variable(base_node.properties.get('api_input_field_list', [])) - workflow_variable = {**default_global_variable, **default_api_global_variable, **get_global_variable(self)} - chat_user_id = workflow_variable.get('chat_user_id') - long_term_memory = None - if chat_user_id: - long_term_memory = QuerySet(ApplicationLongTermMemory).filter( - chat_user_id=chat_user_id, application_id=self.workflow_params.get('application_id') - ).first() - """ - 开始节点 初始化全局变量 - """ - node_variable = { - 'question': question, - 'image': self.workflow_manage.image_list, - 'document': self.workflow_manage.document_list, - 'audio': self.workflow_manage.audio_list, - 'video': self.workflow_manage.video_list, - 'other': self.workflow_manage.other_list, - 'memory': long_term_memory.memory if long_term_memory else '' - } - workflow_variable['memory'] = node_variable['memory'] - self.workflow_manage.chat_context = self.workflow_manage.get_chat_info().get_chat_variable() - return NodeResult(node_variable, workflow_variable) - - def get_details(self, index: int, **kwargs): - global_fields = [] - for field in self.node.properties.get('config')['globalFields']: - key = field['value'] - global_fields.append({ - 'label': field['label'], - 'key': key, - 'value': self.workflow_manage.context[key] if key in self.workflow_manage.context else '' - }) - return { - 'name': self.node.properties.get('stepName'), - "index": index, - "question": self.context.get('question'), - 'run_time': self.context.get('run_time'), - 'type': self.node.type, - 'status': self.status, - 'err_message': self.err_message, - 'image_list': self.context.get('image'), - 'video_list': self.context.get('video'), - 'document_list': self.context.get('document'), - 'audio_list': self.context.get('audio'), - 'other_list': self.context.get('other'), - 'global_fields': global_fields, - 'enableException': self.node.properties.get('enableException'), - } diff --git a/apps/application/flow/step_node/text_to_speech_step_node/__init__.py b/apps/application/flow/step_node/text_to_speech_step_node/__init__.py deleted file mode 100644 index f3feecc9ce2..00000000000 --- a/apps/application/flow/step_node/text_to_speech_step_node/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# coding=utf-8 - -from .impl import * diff --git a/apps/application/flow/step_node/text_to_speech_step_node/i_text_to_speech_node.py b/apps/application/flow/step_node/text_to_speech_step_node/i_text_to_speech_node.py deleted file mode 100644 index 0dde27fea51..00000000000 --- a/apps/application/flow/step_node/text_to_speech_step_node/i_text_to_speech_node.py +++ /dev/null @@ -1,42 +0,0 @@ -# coding=utf-8 - -from typing import Type - -from rest_framework import serializers - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import INode, NodeResult - -from django.utils.translation import gettext_lazy as _ - - -class TextToSpeechNodeSerializer(serializers.Serializer): - tts_model_id = serializers.CharField(required=False, allow_blank=True, allow_null=True, label=_("Model id")) - tts_model_id_type = serializers.CharField(required=False, default='custom', label=_("Model id type")) - tts_model_id_reference = serializers.ListField(required=False, child=serializers.CharField(), allow_empty=True, - label=_("Reference Field")) - is_result = serializers.BooleanField(required=False, - label=_('Whether to return content')) - - content_list = serializers.ListField(required=True, label=_("Text content")) - model_params_setting = serializers.DictField(required=False, - label=_("Model parameter settings")) - - -class ITextToSpeechNode(INode): - type = 'text-to-speech-node' - support = [WorkflowMode.APPLICATION, WorkflowMode.APPLICATION_LOOP, WorkflowMode.KNOWLEDGE, - WorkflowMode.KNOWLEDGE_LOOP, WorkflowMode.TOOL, WorkflowMode.TOOL_LOOP] - - def get_node_params_serializer_class(self) -> Type[serializers.Serializer]: - return TextToSpeechNodeSerializer - - def _run(self): - content = self.workflow_manage.get_reference_field(self.node_params_serializer.data.get('content_list')[0], - self.node_params_serializer.data.get('content_list')[1:]) - return self.execute(content=content, **self.node_params_serializer.data, **self.flow_params_serializer.data) - - def execute(self, tts_model_id, - content, model_params_setting=None, tts_model_id_type=None, tts_model_id_reference=None, - **kwargs) -> NodeResult: - pass diff --git a/apps/application/flow/step_node/text_to_speech_step_node/impl/__init__.py b/apps/application/flow/step_node/text_to_speech_step_node/impl/__init__.py deleted file mode 100644 index 385b9718f6e..00000000000 --- a/apps/application/flow/step_node/text_to_speech_step_node/impl/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# coding=utf-8 - -from .base_text_to_speech_node import BaseTextToSpeechNode diff --git a/apps/application/flow/step_node/text_to_speech_step_node/impl/base_text_to_speech_node.py b/apps/application/flow/step_node/text_to_speech_step_node/impl/base_text_to_speech_node.py deleted file mode 100644 index 861f27312aa..00000000000 --- a/apps/application/flow/step_node/text_to_speech_step_node/impl/base_text_to_speech_node.py +++ /dev/null @@ -1,178 +0,0 @@ -# coding=utf-8 -import io -import mimetypes - -from django.core.files.uploadedfile import InMemoryUploadedFile - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import NodeResult -from application.flow.step_node.text_to_speech_step_node.i_text_to_speech_node import ITextToSpeechNode -from common.utils.common import _remove_empty_lines -from knowledge.models import FileSourceType -from models_provider.tools import get_model_instance_by_model_workspace_id -from oss.serializers.file import FileSerializer -from pydub import AudioSegment - - -def bytes_to_uploaded_file(file_bytes, file_name="generated_audio.mp3"): - content_type, _ = mimetypes.guess_type(file_name) - if content_type is None: - # 如果未能识别，设置为默认的二进制文件类型 - content_type = "application/octet-stream" - # 创建一个内存中的字节流对象 - file_stream = io.BytesIO(file_bytes) - - # 获取文件大小 - file_size = len(file_bytes) - - uploaded_file = InMemoryUploadedFile( - file=file_stream, - field_name=None, - name=file_name, - content_type=content_type, - size=file_size, - charset=None, - ) - return uploaded_file - - -class BaseTextToSpeechNode(ITextToSpeechNode): - def save_context(self, details, workflow_manage): - self.context['answer'] = details.get('answer') - self.context['result'] = details.get('result') - self.context['exception_message'] = details.get('err_message') - if self.node_params.get('is_result', False): - self.answer_text = details.get('answer') - - def execute(self, tts_model_id, - content, model_params_setting=None, tts_model_id_type=None, tts_model_id_reference=None, - max_length=1024, **kwargs) -> NodeResult: - # 处理引用类型 - if tts_model_id_type == 'reference' and tts_model_id_reference: - reference_data = self.workflow_manage.get_reference_field( - tts_model_id_reference[0], - tts_model_id_reference[1:], - ) - if reference_data and isinstance(reference_data, dict): - tts_model_id = reference_data.get('tts_model_id', reference_data.get('model_id', tts_model_id)) - model_params_setting = reference_data.get('model_params_setting') - - from django.utils.translation import gettext_lazy as _ - - if tts_model_id is None or tts_model_id == '': - raise Exception(_('Model is not allowed to be empty')) - # 分割文本为合理片段 - content = _remove_empty_lines(content) - content_chunks = [content[i:i + max_length] - for i in range(0, len(content), max_length)] - - # 生成并收集所有音频片段 - audio_segments = [] - temp_files = [] - - for i, chunk in enumerate(content_chunks): - self.context['content'] = chunk - workspace_id = self.workflow_manage.get_body().get('workspace_id') - model = get_model_instance_by_model_workspace_id( - tts_model_id, workspace_id, **(model_params_setting or {})) - - audio_byte = model.text_to_speech(chunk) - - # 保存为临时音频文件用于合并 - temp_file = io.BytesIO(audio_byte) - audio_segment = AudioSegment.from_file(temp_file) - audio_segments.append(audio_segment) - temp_files.append(temp_file) - - # 合并所有音频片段 - combined_audio = AudioSegment.empty() - for segment in audio_segments: - combined_audio += segment - - # 将合并后的音频转为字节流 - output_buffer = io.BytesIO() - combined_audio.export(output_buffer, format="mp3") - combined_bytes = output_buffer.getvalue() - file_name = 'combined_audio.mp3' - file = bytes_to_uploaded_file(combined_bytes, file_name) - # 存储合并后的音频文件 - file_url = self.upload_file(file) - # 生成音频标签 - audio_label = f'' - file_id = file_url.split('/')[-1] - audio_list = [{'file_id': file_id, 'file_name': file_name, 'url': file_url}] - - # 关闭所有临时文件 - for temp_file in temp_files: - temp_file.close() - output_buffer.close() - - return NodeResult({ - 'answer': audio_label, - 'result': audio_list - }, {}) - - def upload_file(self, file): - if [WorkflowMode.KNOWLEDGE, WorkflowMode.KNOWLEDGE_LOOP].__contains__( - self.workflow_manage.flow.workflow_mode): - return self.upload_knowledge_file(file) - if [WorkflowMode.TOOL, WorkflowMode.TOOL_LOOP].__contains__(self.workflow_manage.flow.workflow_mode): - return self.upload_tool_file(file) - return self.upload_application_file(file) - - def upload_knowledge_file(self, file): - knowledge_id = self.workflow_params.get('knowledge_id') - meta = { - 'debug': False, - 'knowledge_id': knowledge_id, - } - file_url = FileSerializer(data={ - 'file': file, - 'meta': meta, - 'source_id': knowledge_id, - 'source_type': FileSourceType.KNOWLEDGE.value - }).upload() - return file_url - - def upload_tool_file(self, file): - tool_id = self.workflow_params.get('tool_id') - meta = { - 'debug': False, - 'tool_id': tool_id, - } - file_url = FileSerializer(data={ - 'file': file, - 'meta': meta, - 'source_id': tool_id, - 'source_type': FileSourceType.TOOL.value - }).upload() - return file_url - - def upload_application_file(self, file): - application_id = self.workflow_manage.work_flow_post_handler.chat_info.application_id - chat_id = self.workflow_params.get('chat_id') - meta = { - 'debug': False if application_id else True, - 'chat_id': chat_id, - 'application_id': str(application_id) if application_id else None, - } - file_url = FileSerializer(data={ - 'file': file, - 'meta': meta, - 'source_id': meta['application_id'], - 'source_type': FileSourceType.APPLICATION.value - }).upload() - return file_url - - def get_details(self, index: int, **kwargs): - return { - 'name': self.node.properties.get('stepName'), - "index": index, - 'run_time': self.context.get('run_time'), - 'type': self.node.type, - 'status': self.status, - 'content': self.context.get('content'), - 'err_message': self.err_message, - 'answer': self.context.get('answer'), - 'enableException': self.node.properties.get('enableException'), - } diff --git a/apps/application/flow/step_node/text_to_video_step_node/__init__.py b/apps/application/flow/step_node/text_to_video_step_node/__init__.py deleted file mode 100644 index f3feecc9ce2..00000000000 --- a/apps/application/flow/step_node/text_to_video_step_node/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# coding=utf-8 - -from .impl import * diff --git a/apps/application/flow/step_node/text_to_video_step_node/i_text_to_video_node.py b/apps/application/flow/step_node/text_to_video_step_node/i_text_to_video_node.py deleted file mode 100644 index cf0f0252332..00000000000 --- a/apps/application/flow/step_node/text_to_video_step_node/i_text_to_video_node.py +++ /dev/null @@ -1,57 +0,0 @@ -# coding=utf-8 - -from typing import Type - -from django.utils.translation import gettext_lazy as _ -from rest_framework import serializers - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import INode, NodeResult - - -class TextToVideoNodeSerializer(serializers.Serializer): - model_id = serializers.CharField(required=False, allow_blank=True, allow_null=True, label=_("Model id")) - model_id_type = serializers.CharField(required=False, default='custom', label=_("Model id type")) - model_id_reference = serializers.ListField(required=False, child=serializers.CharField(), allow_empty=True, - label=_("Reference Field")) - prompt = serializers.CharField(required=True, label=_("Prompt word (positive)")) - - negative_prompt = serializers.CharField(required=False, label=_("Prompt word (negative)"), - allow_null=True, allow_blank=True, ) - # 多轮对话数量 - dialogue_number = serializers.IntegerField(required=False, default=0, - label=_("Number of multi-round conversations")) - - dialogue_type = serializers.CharField(required=False, default='NODE', - label=_("Conversation storage type")) - - is_result = serializers.BooleanField(required=False, - label=_('Whether to return content')) - - model_params_setting = serializers.JSONField(required=False, default=dict, - label=_("Model parameter settings")) - - -class ITextToVideoNode(INode): - type = 'text-to-video-node' - support = [WorkflowMode.APPLICATION, WorkflowMode.APPLICATION_LOOP, WorkflowMode.KNOWLEDGE, - WorkflowMode.KNOWLEDGE_LOOP, WorkflowMode.TOOL, WorkflowMode.TOOL_LOOP] - - def get_node_params_serializer_class(self) -> Type[serializers.Serializer]: - return TextToVideoNodeSerializer - - def _run(self): - if [WorkflowMode.KNOWLEDGE, WorkflowMode.KNOWLEDGE_LOOP, WorkflowMode.TOOL, - WorkflowMode.TOOL_LOOP].__contains__( - self.workflow_manage.flow.workflow_mode): - return self.execute(**self.node_params_serializer.data, **self.flow_params_serializer.data, - **{'history_chat_record': [], 'stream': True, 'chat_id': None, 'chat_record_id': None}) - else: - return self.execute(**self.node_params_serializer.data, **self.flow_params_serializer.data) - - def execute(self, model_id, prompt, negative_prompt, dialogue_number, dialogue_type, history_chat_record, - model_params_setting, - chat_record_id, - model_id_type=None, model_id_reference=None, - **kwargs) -> NodeResult: - pass diff --git a/apps/application/flow/step_node/text_to_video_step_node/impl/__init__.py b/apps/application/flow/step_node/text_to_video_step_node/impl/__init__.py deleted file mode 100644 index be03d57a2fa..00000000000 --- a/apps/application/flow/step_node/text_to_video_step_node/impl/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# coding=utf-8 - -from .base_text_to_video_node import BaseTextToVideoNode diff --git a/apps/application/flow/step_node/text_to_video_step_node/impl/base_text_to_video_node.py b/apps/application/flow/step_node/text_to_video_step_node/impl/base_text_to_video_node.py deleted file mode 100644 index af787235dbe..00000000000 --- a/apps/application/flow/step_node/text_to_video_step_node/impl/base_text_to_video_node.py +++ /dev/null @@ -1,188 +0,0 @@ -# coding=utf-8 -from functools import reduce -from typing import List - -import requests -from langchain_core.messages import BaseMessage, HumanMessage, AIMessage - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import NodeResult -from application.flow.step_node.text_to_video_step_node.i_text_to_video_node import ITextToVideoNode -from common.utils.common import bytes_to_uploaded_file -from knowledge.models import FileSourceType -from oss.serializers.file import FileSerializer -from models_provider.tools import get_model_instance_by_model_workspace_id -from django.utils.translation import gettext_lazy as _, gettext - - -class BaseTextToVideoNode(ITextToVideoNode): - def save_context(self, details, workflow_manage): - self.context['answer'] = details.get('answer') - self.context['exception_message'] = details.get('err_message') - self.context['question'] = details.get('question') - if self.node_params.get('is_result', False): - self.answer_text = details.get('answer') - - def execute(self, model_id, prompt, negative_prompt, dialogue_number, dialogue_type, history_chat_record, - model_params_setting, - chat_record_id, - model_id_type=None, model_id_reference=None, - **kwargs) -> NodeResult: - # 处理引用类型 - if model_id_type == 'reference' and model_id_reference: - reference_data = self.workflow_manage.get_reference_field( - model_id_reference[0], - model_id_reference[1:], - ) - if reference_data and isinstance(reference_data, dict): - model_id = reference_data.get('model_id', model_id) - model_params_setting = reference_data.get('model_params_setting') - - if model_id is None or model_id == '': - raise Exception(_('Model is not allowed to be empty')) - workspace_id = self.workflow_manage.get_body().get('workspace_id') - ttv_model = get_model_instance_by_model_workspace_id(model_id, workspace_id, - **(model_params_setting or {})) - history_message = self.get_history_message(history_chat_record, dialogue_number) - self.context['history_message'] = history_message - question = self.generate_prompt_question(prompt) - self.context['question'] = question - message_list = self.generate_message_list(question, history_message) - self.context['message_list'] = message_list - self.context['dialogue_type'] = dialogue_type - self.context['negative_prompt'] = self.generate_prompt_question(negative_prompt) - video_urls = ttv_model.generate_video(question, negative_prompt) - # 保存图片 - if video_urls is None: - return NodeResult({'answer': gettext('Failed to generate video')}, {}) - file_name = 'generated_video.mp4' - if isinstance(video_urls, str) and video_urls.startswith('http'): - video_urls = requests.get(video_urls).content - file = bytes_to_uploaded_file(video_urls, file_name) - file_url = self.upload_file(file) - video_label = f'' - video_list = [{'file_id': file_url.split('/')[-1], 'file_name': file_name, 'url': file_url}] - return NodeResult({'answer': video_label, 'chat_model': ttv_model, 'message_list': message_list, - 'video': video_list, - 'history_message': history_message, 'question': question}, {}) - - def upload_file(self, file): - if [WorkflowMode.KNOWLEDGE, WorkflowMode.KNOWLEDGE_LOOP].__contains__( - self.workflow_manage.flow.workflow_mode): - return self.upload_knowledge_file(file) - if [WorkflowMode.TOOL, WorkflowMode.TOOL_LOOP].__contains__(self.workflow_manage.flow.workflow_mode): - return self.upload_tool_file(file) - return self.upload_application_file(file) - - def upload_knowledge_file(self, file): - knowledge_id = self.workflow_params.get('knowledge_id') - meta = { - 'debug': False, - 'knowledge_id': knowledge_id - } - file_url = FileSerializer(data={ - 'file': file, - 'meta': meta, - 'source_id': knowledge_id, - 'source_type': FileSourceType.KNOWLEDGE.value - }).upload() - return file_url - - def upload_tool_file(self, file): - tool_id = self.workflow_params.get('tool_id') - meta = { - 'debug': False, - 'tool_id': tool_id, - } - file_url = FileSerializer(data={ - 'file': file, - 'meta': meta, - 'source_id': tool_id, - 'source_type': FileSourceType.TOOL.value - }).upload() - return file_url - - def upload_application_file(self, file): - application_id = self.workflow_manage.work_flow_post_handler.chat_info.application_id - chat_id = self.workflow_params.get('chat_id') - meta = { - 'debug': False if application_id else True, - 'chat_id': chat_id, - 'application_id': str(application_id) if application_id else None, - } - file_url = FileSerializer(data={ - 'file': file, - 'meta': meta, - 'source_id': meta['application_id'], - 'source_type': FileSourceType.APPLICATION.value - }).upload() - return file_url - - def generate_history_ai_message(self, chat_record): - for val in chat_record.details.values(): - if self.node.id == val['node_id'] and 'image_list' in val: - if val['dialogue_type'] == 'WORKFLOW': - return chat_record.get_ai_message() - image_list = val['image_list'] - return AIMessage(content=[ - *[{'type': 'image_url', 'image_url': {'url': f'{file_url}'}} for file_url in image_list] - ]) - return chat_record.get_ai_message() - - def get_history_message(self, history_chat_record, dialogue_number): - start_index = len(history_chat_record) - dialogue_number - history_message = reduce(lambda x, y: [*x, *y], [ - [self.generate_history_human_message(history_chat_record[index]), - self.generate_history_ai_message(history_chat_record[index])] - for index in - range(start_index if start_index > 0 else 0, len(history_chat_record))], []) - return history_message - - def generate_history_human_message(self, chat_record): - - for data in chat_record.details.values(): - if self.node.id == data['node_id'] and 'image_list' in data: - image_list = data['image_list'] - if len(image_list) == 0 or data['dialogue_type'] == 'WORKFLOW': - return HumanMessage(content=chat_record.problem_text) - return HumanMessage(content=data['question']) - return HumanMessage(content=chat_record.problem_text) - - def generate_prompt_question(self, prompt): - return self.workflow_manage.generate_prompt(prompt) - - def generate_message_list(self, question: str, history_message): - return [ - *history_message, - question - ] - - @staticmethod - def reset_message_list(message_list: List[BaseMessage], answer_text): - result = [{'role': 'user' if isinstance(message, HumanMessage) else 'ai', 'content': message.content} for - message - in - message_list] - result.append({'role': 'ai', 'content': answer_text}) - return result - - def get_details(self, index: int, **kwargs): - return { - 'name': self.node.properties.get('stepName'), - "index": index, - 'run_time': self.context.get('run_time'), - 'history_message': [{'content': message.content, 'role': message.type} for message in - (self.context.get('history_message') if self.context.get( - 'history_message') is not None else [])], - 'question': self.context.get('question'), - 'answer': self.context.get('answer'), - 'type': self.node.type, - 'message_tokens': self.context.get('message_tokens'), - 'answer_tokens': self.context.get('answer_tokens'), - 'status': self.status, - 'err_message': self.err_message, - 'image_list': self.context.get('image_list'), - 'dialogue_type': self.context.get('dialogue_type'), - 'negative_prompt': self.context.get('negative_prompt'), - 'enableException': self.node.properties.get('enableException'), - } diff --git a/apps/application/flow/step_node/tool_lib_node/__init__.py b/apps/application/flow/step_node/tool_lib_node/__init__.py deleted file mode 100644 index 7422965c365..00000000000 --- a/apps/application/flow/step_node/tool_lib_node/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎 - @file： __init__.py - @date：2024/8/8 17:45 - @desc: -""" -from .impl import * \ No newline at end of file diff --git a/apps/application/flow/step_node/tool_lib_node/i_tool_lib_node.py b/apps/application/flow/step_node/tool_lib_node/i_tool_lib_node.py deleted file mode 100644 index 08f3e3a845d..00000000000 --- a/apps/application/flow/step_node/tool_lib_node/i_tool_lib_node.py +++ /dev/null @@ -1,54 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎 - @file： i_function_lib_node.py - @date：2024/8/8 16:21 - @desc: -""" -from typing import Type - -from django.db import connection -from django.db.models import QuerySet -from django.utils.translation import gettext_lazy as _ -from rest_framework import serializers - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import INode, NodeResult -from common.field.common import ObjectField -from tools.models.tool import Tool - - -class InputField(serializers.Serializer): - name = serializers.CharField(required=True, label=_('Variable Name')) - value = ObjectField(required=True, label=_("Variable Value"), model_type_list=[str, list]) - - -class FunctionLibNodeParamsSerializer(serializers.Serializer): - tool_lib_id = serializers.UUIDField(required=True, label=_('Library ID')) - input_field_list = InputField(required=True, many=True) - is_result = serializers.BooleanField(required=False, - label=_('Whether to return content')) - - def is_valid(self, *, raise_exception=False): - super().is_valid(raise_exception=True) - f_lib = QuerySet(Tool).filter(id=self.data.get('tool_lib_id')).first() - # 归还链接到连接池 - connection.close() - if f_lib is None: - raise Exception(_('The function has been deleted')) - - -class IToolLibNode(INode): - type = 'tool-lib-node' - support = [WorkflowMode.APPLICATION, WorkflowMode.APPLICATION_LOOP, WorkflowMode.KNOWLEDGE, - WorkflowMode.KNOWLEDGE_LOOP, WorkflowMode.TOOL, WorkflowMode.TOOL_LOOP] - - def get_node_params_serializer_class(self) -> Type[serializers.Serializer]: - return FunctionLibNodeParamsSerializer - - def _run(self): - return self.execute(**self.node_params_serializer.data, **self.flow_params_serializer.data) - - def execute(self, tool_lib_id, input_field_list, **kwargs) -> NodeResult: - pass diff --git a/apps/application/flow/step_node/tool_lib_node/impl/__init__.py b/apps/application/flow/step_node/tool_lib_node/impl/__init__.py deleted file mode 100644 index c6c0d832175..00000000000 --- a/apps/application/flow/step_node/tool_lib_node/impl/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎 - @file： __init__.py - @date：2024/8/8 17:48 - @desc: -""" -from .base_tool_lib_node import BaseToolLibNodeNode diff --git a/apps/application/flow/step_node/tool_lib_node/impl/base_tool_lib_node.py b/apps/application/flow/step_node/tool_lib_node/impl/base_tool_lib_node.py deleted file mode 100644 index 3cd056b9534..00000000000 --- a/apps/application/flow/step_node/tool_lib_node/impl/base_tool_lib_node.py +++ /dev/null @@ -1,311 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎 - @file： base_function_lib_node.py - @date：2024/8/8 17:49 - @desc: -""" - -import base64 -import io -import json -import mimetypes -import time -import traceback -from typing import Dict - -import uuid_utils.compat as uuid -from django.core.files.uploadedfile import InMemoryUploadedFile -from django.db.models import QuerySet -from django.utils.translation import gettext as _ - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import NodeResult -from application.flow.step_node.tool_lib_node.i_tool_lib_node import IToolLibNode -from common.database_model_manage.database_model_manage import DatabaseModelManage -from common.exception.app_exception import AppApiException -from common.utils.common import common_convert_value -from common.utils.logger import maxkb_logger -from common.utils.rsa_util import rsa_long_decrypt -from common.utils.tool_code import ToolExecutor -from knowledge.models import FileSourceType -from knowledge.models.knowledge_action import State -from oss.serializers.file import FileSerializer -from tools.models import Tool, ToolRecord, ToolTaskTypeChoices - -function_executor = ToolExecutor() - - -def write_context(step_variable: Dict, global_variable: Dict, node, workflow): - if step_variable is not None: - for key in step_variable: - node.context[key] = step_variable[key] - if workflow.is_result(node, NodeResult(step_variable, global_variable)) and 'result' in step_variable: - result = str(step_variable['result']) + '\n' - yield result - node.answer_text = result - node.context['run_time'] = time.time() - node.context['start_time'] - - -def get_field_value(debug_field_list, name, is_required): - result = [field for field in debug_field_list if field.get('name') == name] - if len(result) > 0: - return result[-1]['value'] - if is_required: - raise AppApiException(500, _('Field: {name} No value set').format(name=name)) - return None - - -def valid_reference_value(_type, value, name): - if _type == 'int': - instance_type = int | float - elif _type == 'boolean': - instance_type = bool - elif _type == 'float': - instance_type = float | int - elif _type == 'dict': - value = json.loads(value) if isinstance(value, str) else value - instance_type = dict - elif _type == 'array': - value = json.loads(value) if isinstance(value, str) else value - instance_type = list - elif _type == 'string': - instance_type = str - else: - maxkb_logger.error(_( - 'Field: {name} Type: {_type} Value: {value} Unsupported this type' - ).format(name=name, _type=_type, value=value)) - return value - if not isinstance(value, instance_type): - raise Exception(_( - 'Field: {name} Type: {_type} Value: {value} Type error' - ).format(name=name, _type=_type, value=value)) - return value - - -def convert_value(name: str, value, _type, is_required, source, node): - if not is_required and (value is None or ((isinstance(value, str) or isinstance(value, list)) and len(value) == 0)): - return None - if source == 'reference': - value = node.workflow_manage.get_reference_field( - value[0], - value[1:]) - if value is None: - if not is_required: - return None - else: - raise Exception(_( - 'Field: {name} Type: {_type} is required' - ).format(name=name, _type=_type)) - value = valid_reference_value(_type, value, name) - if _type == 'int': - return int(value) - if _type == 'float': - return float(value) - return value - try: - value = node.workflow_manage.generate_prompt(value) - return common_convert_value(_type, value) - except Exception as e: - raise Exception( - _('Field: {name} Type: {_type} Value: {value} Type error').format(name=name, _type=_type, - value=value)) - - -def valid_function(tool_lib, workspace_id): - if tool_lib is None: - raise Exception(_('Tool does not exist')) - get_authorized_tool = DatabaseModelManage.get_model("get_authorized_tool") - if tool_lib and tool_lib.workspace_id != workspace_id and get_authorized_tool is not None: - tool_lib = get_authorized_tool(QuerySet(Tool).filter(id=tool_lib.id), workspace_id).first() - if tool_lib is None: - raise Exception(_("Tool does not exist")) - if not tool_lib.is_active: - raise Exception(_("Tool is not active")) - - -def _filter_file_bytes(data): - """递归过滤掉所有层级的 file_bytes""" - if isinstance(data, dict): - return {k: _filter_file_bytes(v) for k, v in data.items() if k != 'file_bytes'} - elif isinstance(data, list): - return [_filter_file_bytes(item) for item in data] - else: - return data - - -def bytes_to_uploaded_file(file_bytes, file_name="unknown"): - content_type, _ = mimetypes.guess_type(file_name) - if content_type is None: - # 如果未能识别，设置为默认的二进制文件类型 - content_type = "application/octet-stream" - # 创建一个内存中的字节流对象 - file_stream = io.BytesIO(file_bytes) - - # 获取文件大小 - file_size = len(file_bytes) - - uploaded_file = InMemoryUploadedFile( - file=file_stream, - field_name=None, - name=file_name, - content_type=content_type, - size=file_size, - charset=None, - ) - return uploaded_file - - -def _get_result_detail(result): - if isinstance(result, dict): - result_dict = {k: (str(v)[:500] if len(str(v)) > 500 else v) for k, v in result.items()} - elif isinstance(result, list): - result_dict = [str(item)[:500] if len(str(item)) > 500 else item for item in result] - elif isinstance(result, str): - result_dict = result[:500] if len(result) > 500 else result - else: - result_dict = result - return result_dict - - -class BaseToolLibNodeNode(IToolLibNode): - def save_context(self, details, workflow_manage): - self.context['result'] = details.get('result') - self.context['exception_message'] = details.get('err_message') - if self.node_params.get('is_result'): - self.answer_text = str(details.get('result')) - - def execute(self, tool_lib_id, input_field_list, **kwargs) -> NodeResult: - workspace_id = self.workflow_manage.get_body().get('workspace_id') - tool_lib = QuerySet(Tool).filter(id=tool_lib_id).first() - valid_function(tool_lib, workspace_id) - params = { - field.get('name'): convert_value( - field.get('name'), field.get('value'), field.get('type'), - field.get('is_required'), - field.get('source'), self - ) - for field in [ - { - 'value': get_field_value(input_field_list, field.get('name'), field.get('is_required'), ), **field - } for field in tool_lib.input_field_list - ] - } - - self.context['params'] = params - # 合并初始化参数 - init_params_default_value = {i["field"]: i.get('default_value') for i in tool_lib.init_field_list} - if tool_lib.init_params is not None: - all_params = init_params_default_value | json.loads(rsa_long_decrypt(tool_lib.init_params)) | params - else: - all_params = init_params_default_value | params - if self.node.properties.get('kind') == 'data-source': - exist = function_executor.exec_code( - f'{tool_lib.code}\ndef function_exist(function_name): return callable(globals().get(function_name))', - {'function_name': 'get_download_file_list'}) - all_params = {**all_params, **self.workflow_params.get('data_source')} - if exist: - download_file_list = [] - download_list = function_executor.exec_code(tool_lib.code, - all_params, - function_name='get_download_file_list') - for item in download_list: - result = function_executor.exec_code(tool_lib.code, - {**all_params, 'download_item': item}, - function_name='download') - file_bytes = result.get('file_bytes', []) - chunks = [] - for chunk in file_bytes: - chunks.append(base64.b64decode(chunk)) - file = bytes_to_uploaded_file(b''.join(chunks), result.get('name')) - file_url = self.upload_knowledge_file(file) - download_file_list.append({'file_id': file_url.split('/')[-1], 'name': result.get('name')}) - result = download_file_list - else: - result = function_executor.exec_code(tool_lib.code, all_params) - else: - result = self.tool_exec_record(tool_lib, all_params) - return NodeResult({'result': result}, - (self.workflow_manage.params.get('knowledge_base') or {}) if self.node.properties.get( - 'kind') == 'data-source' else {}, _write_context=write_context) - - def tool_exec_record(self, tool_lib, all_params): - task_record_id = uuid.uuid7() - start_time = time.time() - filtered_args = all_params - try: - # 过滤掉 tool_init_params 中的参数 - tool_init_params = json.loads(rsa_long_decrypt(tool_lib.init_params)) if tool_lib.init_params else {} - if tool_init_params: - filtered_args = { - k: v for k, v in all_params.items() - if k not in tool_init_params - } - if [WorkflowMode.KNOWLEDGE, WorkflowMode.KNOWLEDGE_LOOP].__contains__( - self.workflow_manage.flow.workflow_mode): - source_id = self.workflow_manage.params.get('knowledge_id') - source_type = ToolTaskTypeChoices.KNOWLEDGE.value - elif [WorkflowMode.TOOL, WorkflowMode.TOOL_LOOP].__contains__(self.workflow_manage.flow.workflow_mode): - source_id = self.workflow_manage.params.get('tool_id') - source_type = ToolTaskTypeChoices.TOOL.value - else: - source_id = self.workflow_manage.params.get('application_id') - source_type = ToolTaskTypeChoices.APPLICATION.value - - ToolRecord( - id=task_record_id, - workspace_id=tool_lib.workspace_id, - tool_id=tool_lib.id, - source_type=source_type, - source_id=source_id, - meta={'input': filtered_args, 'output': {}}, - state=State.STARTED - ).save() - - result = function_executor.exec_code(tool_lib.code, all_params) - result_dict = _get_result_detail(result) - QuerySet(ToolRecord).filter(id=task_record_id).update( - state=State.SUCCESS, - run_time=time.time() - start_time, - meta={'input': filtered_args, 'output': result_dict} - ) - - return result - except Exception as e: - maxkb_logger.error(f"Tool execution error: {traceback.format_exc()}") - QuerySet(ToolRecord).filter(id=task_record_id).update( - state=State.FAILURE, - run_time=time.time() - start_time, - meta={'input': filtered_args, 'output': 'Error: ' + str(e)} - ) - - def upload_knowledge_file(self, file): - knowledge_id = self.workflow_params.get('knowledge_id') - meta = { - 'debug': False, - 'knowledge_id': knowledge_id, - } - file_url = FileSerializer(data={ - 'file': file, - 'meta': meta, - 'source_id': knowledge_id, - 'source_type': FileSourceType.KNOWLEDGE.value - }).upload().replace("./oss/file/", '') - file.close() - return file_url - - def get_details(self, index: int, **kwargs): - result = _filter_file_bytes(self.context.get('result')) - - return { - 'name': self.node.properties.get('stepName'), - "index": index, - "result": result, - "params": self.context.get('params'), - 'run_time': self.context.get('run_time'), - 'type': self.node.type, - 'status': self.status, - 'err_message': self.err_message, - 'enableException': self.node.properties.get('enableException'), - } diff --git a/apps/application/flow/step_node/tool_node/__init__.py b/apps/application/flow/step_node/tool_node/__init__.py deleted file mode 100644 index ebfbe8d8bb4..00000000000 --- a/apps/application/flow/step_node/tool_node/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎 - @file： __init__.py.py - @date：2024/8/13 10:43 - @desc: -""" -from .impl import * \ No newline at end of file diff --git a/apps/application/flow/step_node/tool_node/i_tool_node.py b/apps/application/flow/step_node/tool_node/i_tool_node.py deleted file mode 100644 index 4f8343a67db..00000000000 --- a/apps/application/flow/step_node/tool_node/i_tool_node.py +++ /dev/null @@ -1,66 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎 - @file： i_function_lib_node.py - @date：2024/8/8 16:21 - @desc: -""" -import re -from typing import Type - -from django.core import validators -from django.utils.translation import gettext_lazy as _ -from rest_framework import serializers -from rest_framework.utils.formatting import lazy_format - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import INode, NodeResult -from common.exception.app_exception import AppApiException -from common.field.common import ObjectField - - -class InputField(serializers.Serializer): - name = serializers.CharField(required=True, label=_('Variable Name')) - is_required = serializers.BooleanField(required=True, label=_("Is this field required")) - type = serializers.CharField(required=True, label=_("type"), validators=[ - validators.RegexValidator(regex=re.compile("^string|int|dict|array|float|boolean$"), - message=_("The field only supports string|int|dict|array|float"), code=500) - ]) - source = serializers.CharField(required=True, label=_("source"), validators=[ - validators.RegexValidator(regex=re.compile("^custom|reference$"), - message=_("The field only supports custom|reference"), code=500) - ]) - value = ObjectField(required=True, label=_("Variable Value"), model_type_list=[str, list]) - - def is_valid(self, *, raise_exception=False): - super().is_valid(raise_exception=True) - is_required = self.data.get('is_required') - if is_required and self.data.get('value') is None: - message = lazy_format(_('{field}, this field is required.'), field=self.data.get("name")) - raise AppApiException(500, message) - - -class FunctionNodeParamsSerializer(serializers.Serializer): - input_field_list = InputField(required=True, many=True) - code = serializers.CharField(required=True, label=_("function")) - is_result = serializers.BooleanField(required=False, - label=_('Whether to return content')) - - def is_valid(self, *, raise_exception=False): - super().is_valid(raise_exception=True) - - -class IToolNode(INode): - type = 'tool-node' - support = [WorkflowMode.APPLICATION, WorkflowMode.APPLICATION_LOOP, WorkflowMode.KNOWLEDGE, - WorkflowMode.KNOWLEDGE_LOOP, WorkflowMode.TOOL, WorkflowMode.TOOL_LOOP] - - def get_node_params_serializer_class(self) -> Type[serializers.Serializer]: - return FunctionNodeParamsSerializer - - def _run(self): - return self.execute(**self.node_params_serializer.data, **self.flow_params_serializer.data) - - def execute(self, input_field_list, code, **kwargs) -> NodeResult: - pass diff --git a/apps/application/flow/step_node/tool_node/impl/__init__.py b/apps/application/flow/step_node/tool_node/impl/__init__.py deleted file mode 100644 index 0ef86c3b687..00000000000 --- a/apps/application/flow/step_node/tool_node/impl/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎 - @file： __init__.py.py - @date：2024/8/13 11:19 - @desc: -""" -from .base_tool_node import BaseToolNodeNode diff --git a/apps/application/flow/step_node/tool_node/impl/base_tool_node.py b/apps/application/flow/step_node/tool_node/impl/base_tool_node.py deleted file mode 100644 index e269a2b2ba7..00000000000 --- a/apps/application/flow/step_node/tool_node/impl/base_tool_node.py +++ /dev/null @@ -1,129 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎 - @file： base_function_lib_node.py - @date：2024/8/8 17:49 - @desc: -""" -import json -import time -from typing import Dict - -from django.utils.translation import gettext as _ - -from application.flow.i_step_node import NodeResult -from application.flow.step_node.tool_node.i_tool_node import IToolNode -from common.utils.common import common_convert_value -from common.utils.logger import maxkb_logger -from common.utils.tool_code import ToolExecutor -from maxkb.const import CONFIG - -function_executor = ToolExecutor() - - -def write_context(step_variable: Dict, global_variable: Dict, node, workflow): - if step_variable is not None: - for key in step_variable: - node.context[key] = step_variable[key] - if workflow.is_result(node, NodeResult(step_variable, global_variable)) and 'result' in step_variable: - result = str(step_variable['result']) + '\n' - yield result - node.answer_text = result - node.context['run_time'] = time.time() - node.context['start_time'] - - -def valid_reference_value(_type, value, name): - if _type == 'int': - instance_type = int | float - elif _type == 'boolean': - instance_type = bool - elif _type == 'float': - instance_type = float | int - elif _type == 'dict': - value = json.loads(value) if isinstance(value, str) else value - instance_type = dict - elif _type == 'array': - value = json.loads(value) if isinstance(value, str) else value - instance_type = list - elif _type == 'string': - instance_type = str - else: - maxkb_logger.error(_( - 'Field: {name} Type: {_type} Value: {value} Unsupported this type' - ).format(name=name, _type=_type, value=value)) - return value - if not isinstance(value, instance_type): - raise Exception(_( - 'Field: {name} Type: {_type} Value: {value} Type error' - ).format(name=name, _type=_type, value=value)) - return value - - -def convert_value(name: str, value, _type, is_required, source, node): - if not is_required and (value is None or ((isinstance(value, str) or isinstance(value, list)) and len(value) == 0)): - return None - if source == 'reference': - value = node.workflow_manage.get_reference_field( - value[0], - value[1:]) - if value is None: - if not is_required: - return None - else: - raise Exception(_( - 'Field: {name} Type: {_type} is required' - ).format(name=name, _type=_type)) - value = valid_reference_value(_type, value, name) - if _type == 'int': - return int(value) - if _type == 'float': - return float(value) - return value - try: - value = node.workflow_manage.generate_prompt(value) - return common_convert_value(_type, value) - except Exception as e: - raise Exception( - _('Field: {name} Type: {_type} Value: {value} Type error').format(name=name, _type=_type, - value=value)) - - -class BaseToolNodeNode(IToolNode): - def save_context(self, details, workflow_manage): - self.context['result'] = details.get('result') - self.context['exception_message'] = details.get('err_message') - if self.node_params.get('is_result', False): - self.answer_text = str(details.get('result')) - - def execute(self, input_field_list, code, **kwargs) -> NodeResult: - params = {field.get('name'): convert_value(field.get('name'), field.get('value'), field.get('type'), - field.get('is_required'), field.get('source'), self) - for field in input_field_list} - # 合并启动参数默认值（如果有 init_field_list 定义） - init_field_list = self.node_params.get('init_field_list', []) - if init_field_list: - init_params_default_value = {i["field"]: i.get('default_value') for i in init_field_list} - init_params = kwargs.get('init_params') - if init_params is not None: - all_params = init_params_default_value | init_params | params - else: - all_params = init_params_default_value | params - else: - all_params = params - result = function_executor.exec_code(code, all_params) - self.context['params'] = all_params - return NodeResult({'result': result}, {}, _write_context=write_context) - - def get_details(self, index: int, **kwargs): - return { - 'name': self.node.properties.get('stepName'), - "index": index, - "result": self.context.get('result'), - "params": self.context.get('params'), - 'run_time': self.context.get('run_time'), - 'type': self.node.type, - 'status': self.status, - 'err_message': self.err_message, - 'enableException': self.node.properties.get('enableException'), - } diff --git a/apps/application/flow/step_node/tool_start_node/__init__.py b/apps/application/flow/step_node/tool_start_node/__init__.py deleted file mode 100644 index 98a1afcd904..00000000000 --- a/apps/application/flow/step_node/tool_start_node/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： __init__.py - @date：2024/6/11 15:30 - @desc: -""" -from .impl import * diff --git a/apps/application/flow/step_node/tool_start_node/i_tool_start_node.py b/apps/application/flow/step_node/tool_start_node/i_tool_start_node.py deleted file mode 100644 index ca313277376..00000000000 --- a/apps/application/flow/step_node/tool_start_node/i_tool_start_node.py +++ /dev/null @@ -1,21 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： i_start_node.py - @date：2024/6/3 16:54 - @desc: -""" -from application.flow.common import WorkflowMode -from application.flow.i_step_node import INode, NodeResult - - -class IToolStartNode(INode): - type = 'tool-start-node' - support = [WorkflowMode.TOOL] - - def _run(self): - return self.execute(**self.flow_params_serializer.data) - - def execute(self, **kwargs) -> NodeResult: - pass diff --git a/apps/application/flow/step_node/tool_start_node/impl/__init__.py b/apps/application/flow/step_node/tool_start_node/impl/__init__.py deleted file mode 100644 index 6fcd243dc5c..00000000000 --- a/apps/application/flow/step_node/tool_start_node/impl/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： __init__.py - @date：2024/6/11 15:36 - @desc: -""" -from .base_tool_start_node import BaseToolStartStepNode diff --git a/apps/application/flow/step_node/tool_start_node/impl/base_tool_start_node.py b/apps/application/flow/step_node/tool_start_node/impl/base_tool_start_node.py deleted file mode 100644 index 5b24722f76e..00000000000 --- a/apps/application/flow/step_node/tool_start_node/impl/base_tool_start_node.py +++ /dev/null @@ -1,66 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： base_start_node.py - @date：2024/6/3 17:17 - @desc: -""" -from typing import Type - -from rest_framework import serializers - -from application.flow.i_step_node import NodeResult -from application.flow.step_node.tool_start_node.i_tool_start_node import IToolStartNode - - -class BaseToolStartStepNode(IToolStartNode): - def save_context(self, details, workflow_manage): - base_node = self.workflow_manage.get_base_node() - workflow_variable = {} - self.context['exception_message'] = details.get('err_message') - self.status = details.get('status') - self.err_message = details.get('err_message') - for key, value in workflow_variable.items(): - workflow_manage.context[key] = value - for item in details.get('global_fields', []): - workflow_manage.context[item.get('key')] = item.get('value') - - def get_node_params_serializer_class(self) -> Type[serializers.Serializer]: - pass - - def execute(self, **kwargs) -> NodeResult: - base_node = self.workflow_manage.get_base_node() - global_value = {} - params = self.workflow_manage.get_body() - for item in base_node.properties.get('user_input_field_list', []): - global_value[item.get('field')] = params.get(item.get('field')) - - self.workflow_manage.out_context = { - item.get('field'): None - for item in base_node.properties.get('user_output_field_list', []) - if item.get('default_value', None) is not None - } - return NodeResult({}, global_value) - - def get_details(self, index: int, **kwargs): - global_fields = [] - for field in self.node.properties.get('config')['globalFields']: - key = field['value'] - global_fields.append({ - 'label': field.get('label'), - 'key': key, - 'value': self.workflow_manage.context[key] if key in self.workflow_manage.context else '' - }) - return { - 'name': self.node.properties.get('stepName'), - "index": index, - "question": self.context.get('question'), - 'run_time': self.context.get('run_time'), - 'type': self.node.type, - 'status': self.status, - 'err_message': self.err_message, - 'global_fields': global_fields, - '': '', - 'enableException': self.node.properties.get('enableException'), - } diff --git a/apps/application/flow/step_node/tool_workflow_lib_node/__init__.py b/apps/application/flow/step_node/tool_workflow_lib_node/__init__.py deleted file mode 100644 index d417d531251..00000000000 --- a/apps/application/flow/step_node/tool_workflow_lib_node/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎虎 - @file： __init__.py.py - @date：2026/3/16 13:53 - @desc: -""" -from .impl import * \ No newline at end of file diff --git a/apps/application/flow/step_node/tool_workflow_lib_node/i_tool_workflow_lib_node.py b/apps/application/flow/step_node/tool_workflow_lib_node/i_tool_workflow_lib_node.py deleted file mode 100644 index 82b73d0904b..00000000000 --- a/apps/application/flow/step_node/tool_workflow_lib_node/i_tool_workflow_lib_node.py +++ /dev/null @@ -1,57 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎 - @file： i_function_lib_node.py - @date：2024/8/8 16:21 - @desc: -""" -from typing import Type - -from django.db import connection -from django.db.models import QuerySet -from django.utils.translation import gettext_lazy as _ -from rest_framework import serializers - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import INode, NodeResult -from common.field.common import ObjectField -from tools.models.tool import Tool, ToolType - - -class InputField(serializers.Serializer): - field = serializers.CharField(required=True, label=_('Variable Name')) - label = serializers.CharField(required=True, label=_('Variable Label')) - source = serializers.CharField(required=True, label=_('Variable Source')) - type = serializers.CharField(required=True, label=_('Variable Type')) - value = ObjectField(required=True, label=_("Variable Value"), model_type_list=[str, list, bool, dict, int, float]) - - -class FunctionLibNodeParamsSerializer(serializers.Serializer): - tool_lib_id = serializers.UUIDField(required=True, label=_('Library ID')) - input_field_list = InputField(required=True, many=True) - is_result = serializers.BooleanField(required=False, - label=_('Whether to return content')) - - def is_valid(self, *, raise_exception=False): - super().is_valid(raise_exception=True) - f_lib = QuerySet(Tool).filter(id=self.data.get('tool_lib_id'), tool_type=ToolType.WORKFLOW).first() - # 归还链接到连接池 - connection.close() - if f_lib is None: - raise Exception(_('The function has been deleted')) - - -class IToolWorkflowLibNode(INode): - type = 'tool-workflow-lib-node' - support = [WorkflowMode.APPLICATION, WorkflowMode.APPLICATION_LOOP, WorkflowMode.KNOWLEDGE, - WorkflowMode.KNOWLEDGE_LOOP, WorkflowMode.TOOL, WorkflowMode.TOOL_LOOP] - - def get_node_params_serializer_class(self) -> Type[serializers.Serializer]: - return FunctionLibNodeParamsSerializer - - def _run(self): - return self.execute(**self.node_params_serializer.data, **self.flow_params_serializer.data) - - def execute(self, tool_lib_id, input_field_list, **kwargs) -> NodeResult: - pass diff --git a/apps/application/flow/step_node/tool_workflow_lib_node/impl/__init__.py b/apps/application/flow/step_node/tool_workflow_lib_node/impl/__init__.py deleted file mode 100644 index 0b593554784..00000000000 --- a/apps/application/flow/step_node/tool_workflow_lib_node/impl/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎虎 - @file： __init__.py.py - @date：2026/3/16 13:53 - @desc: -""" -from .base_tool_workflow_lib_node import * diff --git a/apps/application/flow/step_node/tool_workflow_lib_node/impl/base_tool_workflow_lib_node.py b/apps/application/flow/step_node/tool_workflow_lib_node/impl/base_tool_workflow_lib_node.py deleted file mode 100644 index eaaedb28c0a..00000000000 --- a/apps/application/flow/step_node/tool_workflow_lib_node/impl/base_tool_workflow_lib_node.py +++ /dev/null @@ -1,256 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎虎 - @file： base_tool_workflow_lib_node.py.py - @date：2026/3/16 13:55 - @desc: -""" - -import time -from typing import Dict - -import uuid_utils.compat as uuid -from django.db.models import QuerySet -from django.utils.translation import gettext_lazy as _ - -from application.flow.common import WorkflowMode, Workflow, Answer -from application.flow.i_step_node import NodeResult, ToolWorkflowPostHandler, INode -from application.flow.step_node.tool_workflow_lib_node.i_tool_workflow_lib_node import IToolWorkflowLibNode -from application.models import ChatRecord -from application.serializers.common import ToolExecute -from common.database_model_manage.database_model_manage import DatabaseModelManage -from common.exception.app_exception import ChatException -from common.handle.impl.response.loop_to_response import LoopToResponse -from tools.models import ToolWorkflowVersion, Tool - - -def _write_context(node_variable: Dict, workflow_variable: Dict, node: INode, workflow, answer: str, - reasoning_content: str): - result = node_variable.get('result') - node.context['application_node_dict'] = node_variable.get('application_node_dict') - node.context['node_dict'] = node_variable.get('node_dict', {}) - node.context['is_interrupt_exec'] = node_variable.get('is_interrupt_exec') - node.context['message_tokens'] = result.get('usage', {}).get('prompt_tokens', 0) - node.context['answer_tokens'] = result.get('usage', {}).get('completion_tokens', 0) - node.context['answer'] = answer - node.context['result'] = answer - node.context['reasoning_content'] = reasoning_content - node.context['run_time'] = time.time() - node.context['start_time'] - if workflow.is_result(node, NodeResult(node_variable, workflow_variable)): - node.answer_text = answer - - -def get_answer_list(instance, child_node_node_dict, runtime_node_id): - answer_list = instance.get_record_answer_list() - for a in answer_list: - _v = child_node_node_dict.get(a.get('runtime_node_id')) - if _v: - a['runtime_node_id'] = runtime_node_id - a['child_node'] = _v - return answer_list - - -def write_context_stream(node_variable: Dict, workflow_variable: Dict, node: INode, workflow): - """ - 写入上下文数据 (流式) - @param node_variable: 节点数据 - @param workflow_variable: 全局数据 - @param node: 节点 - @param workflow: 工作流管理器 - """ - workflow_manage_new_instance = node_variable.get('workflow_manage_new_instance') - node_params = node.node_params - start_node_id = node_params.get('child_node', {}).get('runtime_node_id') - child_node_data = node.context.get('child_node_data') or [] - start_node_data = None - chat_record = None - child_node = None - if start_node_id: - chat_record_id = node_params.get('child_node', {}).get('chat_record_id') - child_node = node_params.get('child_node', {}).get('child_node') - start_node_data = node_params.get('node_data') - chat_record = ChatRecord(id=chat_record_id, answer_text_list=[], answer_text='', - details=child_node_data) - instance = workflow_manage_new_instance(start_node_id, - start_node_data, chat_record, child_node) - answer = '' - reasoning_content = '' - usage = {} - node_child_node = {} - is_interrupt_exec = False - response = instance.stream() - child_node_node_dict = {} - for chunk in response: - response_content = chunk - content = (response_content.get('content', '') or '') - runtime_node_id = response_content.get('runtime_node_id', '') - chat_record_id = response_content.get('chat_record_id', '') - child_node = response_content.get('child_node') - node_type = response_content.get('node_type') - _reasoning_content = (response_content.get('reasoning_content', '') or '') - if node_type == 'form-node': - is_interrupt_exec = True - answer += content - reasoning_content += _reasoning_content - node_child_node = {'runtime_node_id': runtime_node_id, 'chat_record_id': chat_record_id, - 'child_node': child_node} - - child_node = chunk.get('child_node') - runtime_node_id = chunk.get('runtime_node_id', '') - chat_record_id = chunk.get('chat_record_id', '') - child_node_node_dict[runtime_node_id] = { - 'runtime_node_id': runtime_node_id, - 'chat_record_id': chat_record_id, - 'child_node': child_node} - content_chunk = (chunk.get('content', '') or '') - reasoning_content_chunk = (chunk.get('reasoning_content', '') or '') - reasoning_content += reasoning_content_chunk - answer += content_chunk - yield chunk - if chunk.get('node_status', "SUCCESS") == 'ERROR': - is_interrupt_exec = True - node.status = 500 - node.err_message = chunk.get('content') - usage = response_content.get('usage', {}) - child_answer_data = get_answer_list(instance, child_node_node_dict, node.runtime_node_id) - node.context['usage'] = {'usage': usage} - node.context['child_node'] = node_child_node - node.context['details'] = instance.get_runtime_details() - node.context['is_interrupt_exec'] = is_interrupt_exec - node.context['child_answer_data'] = child_answer_data - node.context['run_time'] = time.time() - node.context.get("start_time") - node.extra['input_field_list'] = instance.get_input_field_list() - node.extra['output_field_list'] = instance.get_output_field_list() - node.extra['input'] = instance.get_input() - node.extra['output'] = instance.out_context - for key, value in instance.out_context.items(): - node.context[key] = value - - -def _is_interrupt_exec(node, node_variable: Dict, workflow_variable: Dict): - return node.context.get('is_interrupt_exec', False) - - -def valid_function(tool_lib, workspace_id): - if tool_lib is None: - raise Exception(_('Tool does not exist')) - get_authorized_tool = DatabaseModelManage.get_model("get_authorized_tool") - if tool_lib and tool_lib.workspace_id != workspace_id and get_authorized_tool is not None: - tool_lib = get_authorized_tool(QuerySet(Tool).filter(id=tool_lib.id), workspace_id).first() - if tool_lib is None: - raise Exception(_("Tool does not exist")) - if not tool_lib.is_active: - raise Exception(_("Tool is not active")) - - -class BaseToolWorkflowLibNodeNode(IToolWorkflowLibNode): - def get_answer_list(self): - try: - child_answer_data = self.context.get('child_answer_data') or [] - return [ - Answer(item.get('content'), item.get('view_type'), item.get('runtime_node_id'), - item.get('chat_record_id'), - item.get('child_node'), item.get('real_node_id'), item.get('reasoning_content')) for item in - child_answer_data] - except Exception as e: - return [] - - def get_parameters(self, input_field_list): - result = {} - for input in input_field_list: - source = input.get('source') - value = input.get('value') - if source == 'reference': - value = self.workflow_manage.get_reference_field( - value[0], - value[1:]) - result[input.get('field')] = value - - return result - - def save_context(self, details, workflow_manage): - self.context['child_answer_data'] = details.get('child_answer_data') - self.context['details'] = details.get('details') - self.extra['input_field_list'] = details.get('input_field_list') - self.extra['output_field_list'] = details.get('output_field_list') - self.extra['input'] = details.get('input') - self.extra['output'] = details.get('output') - self.context['result'] = details.get('result') - self.context['exception_message'] = details.get('err_message') - for key, value in (details.get('output') or {}).items(): - self.context[key] = value - if self.node_params.get('is_result'): - self.answer_text = str(details.get('result')) - - @staticmethod - def to_chat_record(record): - if record is None: - return None - return ChatRecord( - answer_text_list=record.meta.get('answer_text_list'), - details=record.meta.get('details'), - answer_text='', - ) - - def execute(self, tool_lib_id, input_field_list, **kwargs) -> NodeResult: - from application.flow.tool_workflow_manage import ToolWorkflowManage - workspace_id = self.workflow_manage.get_body().get('workspace_id') - tool_workflow_version = QuerySet(ToolWorkflowVersion).filter(tool_id=tool_lib_id).order_by( - '-create_time')[0:1].first() - if tool_workflow_version is None: - raise ChatException(500, _("The tool has not been published. Please use it after publishing.")) - tool_lib = QuerySet(Tool).filter(id=tool_lib_id).first() - valid_function(tool_lib, workspace_id) - parameters = self.get_parameters(input_field_list) - tool_record_id = (self.node_params.get('child_node') or {}).get('chat_record_id') or str(uuid.uuid7()) - took_execute = ToolExecute(tool_lib_id, tool_record_id, - workspace_id, - self.workflow_manage.get_source_type(), - self.workflow_manage.get_source_id(), - False) - - def workflow_manage_new_instance(start_node_id=None, - start_node_data=None, chat_record=None, child_node=None): - work_flow_manage = ToolWorkflowManage( - Workflow.new_instance(tool_workflow_version.work_flow, WorkflowMode.TOOL), - { - 'chat_record_id': tool_record_id, - 'tool_id': tool_lib_id, - 'stream': True, - 'workspace_id': workspace_id, - **parameters}, - ToolWorkflowPostHandler(took_execute, tool_lib_id), - base_to_response=LoopToResponse(), - start_node_id=start_node_id, - start_node_data=start_node_data, - child_node=child_node, - chat_record=self.to_chat_record(took_execute.get_record()), - is_the_task_interrupted=lambda: False) - - return work_flow_manage - - return NodeResult({'workflow_manage_new_instance': workflow_manage_new_instance}, - {}, _write_context=write_context_stream, - _is_interrupt=_is_interrupt_exec) - - def get_details(self, index: int, **kwargs): - result = self.context.get('result') - - return { - 'name': self.node.properties.get('stepName'), - "index": index, - "result": result, - "params": self.context.get('params'), - 'run_time': self.context.get('run_time'), - 'type': self.node.type, - 'status': self.status, - 'input': self.extra.get('input'), - 'output': self.extra.get('output'), - 'input_field_list': self.extra.get('input_field_list'), - 'output_field_list': self.extra.get('output_field_list'), - 'details': self.context.get("details"), - 'child_answer_data': self.context.get("child_answer_data"), - 'err_message': self.err_message, - 'enableException': self.node.properties.get('enableException'), - } diff --git a/apps/application/flow/step_node/variable_aggregation_node/__init__.py b/apps/application/flow/step_node/variable_aggregation_node/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/apps/application/flow/step_node/variable_aggregation_node/i_variable_aggregation_node.py b/apps/application/flow/step_node/variable_aggregation_node/i_variable_aggregation_node.py deleted file mode 100644 index 86a38778292..00000000000 --- a/apps/application/flow/step_node/variable_aggregation_node/i_variable_aggregation_node.py +++ /dev/null @@ -1,42 +0,0 @@ -# coding=utf-8 - -from typing import Type - -from django.utils.translation import gettext_lazy as _ -from rest_framework import serializers - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import INode, NodeResult - - -class VariableListSerializer(serializers.Serializer): - v_id = serializers.CharField(required=True, label=_("Variable id")) - key = serializers.CharField(required=False, label=_("Key"), allow_null=True, allow_blank=True, ) - variable = serializers.ListField(required=True, label=_("Variable")) - - -class VariableGroupSerializer(serializers.Serializer): - id = serializers.CharField(required=True, label=_("Group id")) - field = serializers.CharField(required=True, label=_("group_name")) - label = serializers.CharField(required=True) - variable_list = VariableListSerializer(many=True) - - -class VariableAggregationNodeSerializer(serializers.Serializer): - strategy = serializers.CharField(required=True, label=_("Strategy")) - group_list = VariableGroupSerializer(many=True) - - -class IVariableAggregation(INode): - type = 'variable-aggregation-node' - support = [WorkflowMode.APPLICATION, WorkflowMode.APPLICATION_LOOP, WorkflowMode.KNOWLEDGE, - WorkflowMode.KNOWLEDGE_LOOP, WorkflowMode.TOOL, WorkflowMode.TOOL_LOOP] - - def get_node_params_serializer_class(self) -> Type[serializers.Serializer]: - return VariableAggregationNodeSerializer - - def _run(self): - return self.execute(**self.node_params_serializer.data, **self.flow_params_serializer.data) - - def execute(self, strategy, group_list, **kwargs) -> NodeResult: - pass diff --git a/apps/application/flow/step_node/variable_aggregation_node/impl/__init__.py b/apps/application/flow/step_node/variable_aggregation_node/impl/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/apps/application/flow/step_node/variable_aggregation_node/impl/base_variable_aggregation_node.py b/apps/application/flow/step_node/variable_aggregation_node/impl/base_variable_aggregation_node.py deleted file mode 100644 index 341f2e0eab9..00000000000 --- a/apps/application/flow/step_node/variable_aggregation_node/impl/base_variable_aggregation_node.py +++ /dev/null @@ -1,98 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎² - @file： base_variable_aggregation_node.py - @date：2025/10/23 17:42 - @desc: -""" -from application.flow.i_step_node import NodeResult -from application.flow.step_node.variable_aggregation_node.i_variable_aggregation_node import IVariableAggregation - - -def _filter_file_bytes(data): - """递归过滤掉所有层级的 file_bytes""" - if isinstance(data, dict): - return {k: _filter_file_bytes(v) for k, v in data.items() if k != 'file_bytes'} - elif isinstance(data, list): - return [_filter_file_bytes(item) for item in data] - else: - return data - - -class BaseVariableAggregationNode(IVariableAggregation): - - def save_context(self, details, workflow_manage): - for key, value in details.get('result').items(): - self.context[key] = value - self.context['result'] = details.get('result') - self.context['strategy'] = details.get('strategy') - self.context['group_list'] = details.get('group_list') - self.context['exception_message'] = details.get('err_message') - - def get_first_non_null(self, variable_list): - for variable in variable_list: - v = self.workflow_manage.get_reference_field( - variable.get('variable')[0], - variable.get('variable')[1:]) - if v is not None and not (isinstance(v, (str, list, dict)) and len(v) == 0): - return v - return None - - def set_variable_to_array(self, variable_list): - return [self.workflow_manage.get_reference_field( - variable.get('variable')[0], - variable.get('variable')[1:]) for variable in variable_list] - - def set_variable_to_dict(self, variable_list): - return {(variable.get('key') or variable.get('variable')[-1]): self.workflow_manage.get_reference_field( - variable.get('variable')[0], - variable.get('variable')[1:]) for variable in variable_list} - - def reset_variable(self, variable): - value = self.workflow_manage.get_reference_field( - variable.get('variable')[0], - variable.get('variable')[1:]) - node_id = variable.get('variable')[0] - node = self.workflow_manage.flow.get_node(node_id) - return {"value": value, 'node_name': node.properties.get('stepName') if node is not None else node_id, - 'field': variable.get('variable')[1]} - - def reset_group_list(self, group_list): - result = [] - for g in group_list: - b = {'label': g.get('label'), - 'variable_list': [self.reset_variable(variable) for variable in g.get('variable_list')]} - result.append(b) - return result - - def execute(self, strategy, group_list, **kwargs) -> NodeResult: - strategy_map = {'first_non_null': self.get_first_non_null, - 'variable_to_array': self.set_variable_to_array, - 'variable_to_dict': self.set_variable_to_dict, - } - - # 向下兼容 - if strategy == 'variable_to_json': - strategy = 'variable_to_array' - - result = {item.get('field'): strategy_map[strategy](item.get('variable_list')) for item in group_list} - - return NodeResult( - {'result': result, 'strategy': strategy, 'group_list': self.reset_group_list(group_list), **result}, {}) - - def get_details(self, index: int, **kwargs): - result = _filter_file_bytes(self.context.get('result')) - group_list = _filter_file_bytes(self.context.get('group_list')) - return { - 'name': self.node.properties.get('stepName'), - "index": index, - 'run_time': self.context.get('run_time'), - 'type': self.node.type, - 'result': result, - 'strategy': self.context.get('strategy'), - 'group_list': group_list, - 'status': self.status, - 'err_message': self.err_message, - 'enableException': self.node.properties.get('enableException'), - } diff --git a/apps/application/flow/step_node/variable_assign_node/__init__.py b/apps/application/flow/step_node/variable_assign_node/__init__.py deleted file mode 100644 index 2d231e6066d..00000000000 --- a/apps/application/flow/step_node/variable_assign_node/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# coding=utf-8 - -from .impl import * \ No newline at end of file diff --git a/apps/application/flow/step_node/variable_assign_node/i_variable_assign_node.py b/apps/application/flow/step_node/variable_assign_node/i_variable_assign_node.py deleted file mode 100644 index 6652cbe9e9a..00000000000 --- a/apps/application/flow/step_node/variable_assign_node/i_variable_assign_node.py +++ /dev/null @@ -1,29 +0,0 @@ -# coding=utf-8 - -from typing import Type - -from django.utils.translation import gettext_lazy as _ -from rest_framework import serializers - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import INode, NodeResult - - -class VariableAssignNodeParamsSerializer(serializers.Serializer): - variable_list = serializers.ListField(required=True, - label=_("Reference Field")) - - -class IVariableAssignNode(INode): - type = 'variable-assign-node' - support = [WorkflowMode.APPLICATION, WorkflowMode.APPLICATION_LOOP, WorkflowMode.KNOWLEDGE, - WorkflowMode.KNOWLEDGE_LOOP, WorkflowMode.TOOL, WorkflowMode.TOOL_LOOP] - - def get_node_params_serializer_class(self) -> Type[serializers.Serializer]: - return VariableAssignNodeParamsSerializer - - def _run(self): - return self.execute(**self.node_params_serializer.data, **self.flow_params_serializer.data) - - def execute(self, variable_list, **kwargs) -> NodeResult: - pass diff --git a/apps/application/flow/step_node/variable_assign_node/impl/__init__.py b/apps/application/flow/step_node/variable_assign_node/impl/__init__.py deleted file mode 100644 index 7585cdd8fe4..00000000000 --- a/apps/application/flow/step_node/variable_assign_node/impl/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： __init__.py - @date：2024/6/11 17:49 - @desc: -""" -from .base_variable_assign_node import * \ No newline at end of file diff --git a/apps/application/flow/step_node/variable_assign_node/impl/base_variable_assign_node.py b/apps/application/flow/step_node/variable_assign_node/impl/base_variable_assign_node.py deleted file mode 100644 index b9572805acf..00000000000 --- a/apps/application/flow/step_node/variable_assign_node/impl/base_variable_assign_node.py +++ /dev/null @@ -1,125 +0,0 @@ -# coding=utf-8 -import json -from typing import List - -from application.flow.i_step_node import NodeResult -from application.flow.step_node.variable_assign_node.i_variable_assign_node import IVariableAssignNode - - -class BaseVariableAssignNode(IVariableAssignNode): - def save_context(self, details, workflow_manage): - self.context['variable_list'] = details.get('variable_list') - self.context['result_list'] = details.get('result_list') - self.context['exception_message'] = details.get('err_message') - - def global_evaluation(self, variable, value): - from application.flow.loop_workflow_manage import LoopWorkflowManage - if isinstance(self.workflow_manage, LoopWorkflowManage): - self.workflow_manage.parentWorkflowManage.context[variable['fields'][1]] = value - else: - self.workflow_manage.context[variable['fields'][1]] = value - - def loop_evaluation(self, variable, value): - from application.flow.loop_workflow_manage import LoopWorkflowManage - if isinstance(self.workflow_manage, LoopWorkflowManage): - self.workflow_manage.get_loop_context()[variable['fields'][1]] = value - - def chat_evaluation(self, variable, value): - from application.flow.loop_workflow_manage import LoopWorkflowManage - if isinstance(self.workflow_manage, LoopWorkflowManage): - self.workflow_manage.parentWorkflowManage.chat_context[variable['fields'][1]] = value - else: - self.workflow_manage.chat_context[variable['fields'][1]] = value - - def out_evaluation(self, variable, value): - from application.flow.loop_workflow_manage import LoopWorkflowManage - if isinstance(self.workflow_manage, LoopWorkflowManage): - self.workflow_manage.parentWorkflowManage.out_context[variable['fields'][1]] = value - else: - self.workflow_manage.out_context[variable['fields'][1]] = value - - def handle(self, variable, evaluation): - result = { - 'name': variable['name'], - 'input_value': self.get_reference_content(variable['fields']), - } - if variable['source'] == 'custom': - if variable['type'] == 'json': - if isinstance(variable['value'], dict) or isinstance(variable['value'], list): - val = variable['value'] - else: - val = json.loads(variable['value']) - evaluation(variable, val) - result['output_value'] = variable['value'] = val - elif variable['type'] == 'string': - # 变量解析 例如：{{global.xxx}} - val = self.workflow_manage.generate_prompt(variable['value']) - evaluation(variable, val) - result['output_value'] = val - else: - val = variable['value'] - evaluation(variable, val) - result['output_value'] = val - elif variable['source'] == 'referencing': - reference = self.get_reference_content(variable['reference']) - evaluation(variable, reference) - result['output_value'] = reference - else: - val = None - evaluation(variable, val) - result['output_value'] = val - - # 获取输入输出值的类型，用于显示在执行详情页面中 - result['input_type'] = type(result.get('input_value')).__name__ if result.get('input_value') is not None else 'null' - result['output_type'] = type(result.get('output_value')).__name__ if result.get('output_value') is not None else 'null' - - return result - - def execute(self, variable_list, **kwargs) -> NodeResult: - result_list = [] - contains_chat_variable = False - for variable in variable_list: - if not variable.get('fields'): - continue - - field0 = variable['fields'][0] - if 'global' == field0: - result = self.handle(variable, self.global_evaluation) - result_list.append(result) - elif 'chat' == field0: - result = self.handle(variable, self.chat_evaluation) - result_list.append(result) - contains_chat_variable = True - elif 'loop' == field0: - result = self.handle(variable, self.loop_evaluation) - result_list.append(result) - elif 'output' == field0: - result = self.handle(variable, self.out_evaluation) - result_list.append(result) - - if contains_chat_variable: - from application.flow.loop_workflow_manage import LoopWorkflowManage - if isinstance(self.workflow_manage, LoopWorkflowManage): - self.workflow_manage.parentWorkflowManage.get_chat_info().set_chat_variable( - self.workflow_manage.parentWorkflowManage.chat_context) - else: - self.workflow_manage.get_chat_info().set_chat_variable(self.workflow_manage.chat_context) - return NodeResult({'variable_list': variable_list, 'result_list': result_list}, {}) - - def get_reference_content(self, fields: List[str]): - return self.workflow_manage.get_reference_field( - fields[0], - fields[1:]) if fields else None - - def get_details(self, index: int, **kwargs): - return { - 'name': self.node.properties.get('stepName'), - "index": index, - 'run_time': self.context.get('run_time'), - 'type': self.node.type, - 'variable_list': self.context.get('variable_list'), - 'result_list': self.context.get('result_list'), - 'status': self.status, - 'err_message': self.err_message, - 'enableException': self.node.properties.get('enableException'), - } diff --git a/apps/application/flow/step_node/variable_splitting_node/__init__.py b/apps/application/flow/step_node/variable_splitting_node/__init__.py deleted file mode 100644 index c93d71e9ed1..00000000000 --- a/apps/application/flow/step_node/variable_splitting_node/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎虎 - @file： __init__.py.py - @date：2025/10/13 14:56 - @desc: -""" -from .impl import * diff --git a/apps/application/flow/step_node/variable_splitting_node/i_variable_splitting_node.py b/apps/application/flow/step_node/variable_splitting_node/i_variable_splitting_node.py deleted file mode 100644 index 39c48f817be..00000000000 --- a/apps/application/flow/step_node/variable_splitting_node/i_variable_splitting_node.py +++ /dev/null @@ -1,35 +0,0 @@ -# coding=utf-8 - -from typing import Type - -from django.utils.translation import gettext_lazy as _ -from rest_framework import serializers - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import INode, NodeResult - - -class VariableSplittingNodeParamsSerializer(serializers.Serializer): - input_variable = serializers.ListField(required=True, - label=_("input variable")) - - variable_list = serializers.ListField(required=True, - label=_("Split variables")) - - -class IVariableSplittingNode(INode): - type = 'variable-splitting-node' - support = [WorkflowMode.APPLICATION, WorkflowMode.APPLICATION_LOOP, WorkflowMode.KNOWLEDGE, - WorkflowMode.KNOWLEDGE_LOOP, WorkflowMode.TOOL, WorkflowMode.TOOL_LOOP] - - def get_node_params_serializer_class(self) -> Type[serializers.Serializer]: - return VariableSplittingNodeParamsSerializer - - def _run(self): - input_variable = self.workflow_manage.get_reference_field( - self.node_params_serializer.data.get('input_variable')[0], - self.node_params_serializer.data.get('input_variable')[1:]) - return self.execute(input_variable, self.node_params_serializer.data['variable_list']) - - def execute(self, input_variable, variable_list, **kwargs) -> NodeResult: - pass diff --git a/apps/application/flow/step_node/variable_splitting_node/impl/__init__.py b/apps/application/flow/step_node/variable_splitting_node/impl/__init__.py deleted file mode 100644 index 1ef0d7ac519..00000000000 --- a/apps/application/flow/step_node/variable_splitting_node/impl/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎虎 - @file： __init__.py.py - @date：2025/10/13 15:01 - @desc: -""" -from .base_variable_splitting_node import * \ No newline at end of file diff --git a/apps/application/flow/step_node/variable_splitting_node/impl/base_variable_splitting_node.py b/apps/application/flow/step_node/variable_splitting_node/impl/base_variable_splitting_node.py deleted file mode 100644 index 274604e2328..00000000000 --- a/apps/application/flow/step_node/variable_splitting_node/impl/base_variable_splitting_node.py +++ /dev/null @@ -1,80 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎虎 - @file： base_variable_splitting_node.py - @date：2025/10/13 15:02 - @desc: -""" -import json -from jsonpath_ng.ext import parse -from common.cache.mem_cache import MemCache - -from application.flow.i_step_node import NodeResult -from application.flow.step_node.variable_splitting_node.i_variable_splitting_node import IVariableSplittingNode - -jsonpath_expr_cache = MemCache('parse_path', { - 'TIMEOUT': 3600, # 缓存有效期为 1 小时 - 'OPTIONS': { - 'MAX_ENTRIES': 1000, # 最多缓存 1000 个条目 - 'CULL_FREQUENCY': 10, # 达到上限时，删除约 1/10 的缓存 - }, -}) - -def parse_and_cache(path): - jsonpath_expr = jsonpath_expr_cache.get(path) - if not jsonpath_expr: - jsonpath_expr = parse(path) - jsonpath_expr_cache.set(path, jsonpath_expr) - return jsonpath_expr - -def smart_jsonpath_search(data: dict, path: str): - """ - 智能JSON Path搜索 - 返回: - - 单个匹配: 直接返回值 - - 多个匹配: 返回值的列表 - - 无匹配: 返回None - """ - jsonpath_expr = parse_and_cache(path) - matches = jsonpath_expr.find(data) - - if not matches: - return None - elif len(matches) == 1: - return matches[0].value - else: - return [match.value for match in matches] - - -class BaseVariableSplittingNode(IVariableSplittingNode): - def save_context(self, details, workflow_manage): - for key, value in details.get('result').items(): - self.context[key] = value - self.context['result'] = details.get('result') - self.context['request'] = details.get('request') - self.context['exception_message'] = details.get('err_message') - - def execute(self, input_variable, variable_list, **kwargs) -> NodeResult: - if isinstance(input_variable, str): - try: - input_variable = json.loads(input_variable) - except Exception: - pass - - self.context['request'] = input_variable - response = {v['field']: smart_jsonpath_search(input_variable, v['expression']) for v in variable_list} - return NodeResult({'result': response, **response}, {}) - - def get_details(self, index: int, **kwargs): - return { - 'name': self.node.properties.get('stepName'), - "index": index, - 'run_time': self.context.get('run_time'), - 'type': self.node.type, - 'request': self.context.get('request'), - 'result': self.context.get('result'), - 'status': self.status, - 'err_message': self.err_message, - 'enableException': self.node.properties.get('enableException'), - } diff --git a/apps/application/flow/step_node/video_understand_step_node/__init__.py b/apps/application/flow/step_node/video_understand_step_node/__init__.py deleted file mode 100644 index f3feecc9ce2..00000000000 --- a/apps/application/flow/step_node/video_understand_step_node/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# coding=utf-8 - -from .impl import * diff --git a/apps/application/flow/step_node/video_understand_step_node/i_video_understand_node.py b/apps/application/flow/step_node/video_understand_step_node/i_video_understand_node.py deleted file mode 100644 index 8d854291686..00000000000 --- a/apps/application/flow/step_node/video_understand_step_node/i_video_understand_node.py +++ /dev/null @@ -1,63 +0,0 @@ -# coding=utf-8 - -from typing import Type - -from django.utils.translation import gettext_lazy as _ -from rest_framework import serializers - -from application.flow.common import WorkflowMode -from application.flow.i_step_node import INode, NodeResult - - -class VideoUnderstandNodeSerializer(serializers.Serializer): - model_id = serializers.CharField(required=False, allow_blank=True, allow_null=True, label=_("Model id")) - model_id_type = serializers.CharField(required=False, default='custom', label=_("Model id type")) - model_id_reference = serializers.ListField(required=False, child=serializers.CharField(), allow_empty=True, - label=_("Reference Field")) - system = serializers.CharField(required=False, allow_blank=True, allow_null=True, - label=_("Role Setting")) - prompt = serializers.CharField(required=True, label=_("Prompt word")) - # 多轮对话数量 - dialogue_number = serializers.IntegerField(required=True, label=_("Number of multi-round conversations")) - - dialogue_type = serializers.CharField(required=True, label=_("Conversation storage type")) - - is_result = serializers.BooleanField(required=False, - label=_('Whether to return content')) - - video_list = serializers.ListField(required=False, label=_("video")) - - model_params_setting = serializers.JSONField(required=False, default=dict, - label=_("Model parameter settings")) - model_setting = serializers.DictField(required=False, - label='Model settings') - - -class IVideoUnderstandNode(INode): - type = 'video-understand-node' - support = [WorkflowMode.APPLICATION, WorkflowMode.APPLICATION_LOOP, WorkflowMode.KNOWLEDGE, - WorkflowMode.KNOWLEDGE_LOOP, WorkflowMode.TOOL, WorkflowMode.TOOL_LOOP] - - def get_node_params_serializer_class(self) -> Type[serializers.Serializer]: - return VideoUnderstandNodeSerializer - - def _run(self): - res = self.workflow_manage.get_reference_field(self.node_params_serializer.data.get('video_list')[0], - self.node_params_serializer.data.get('video_list')[1:]) - - if [WorkflowMode.KNOWLEDGE, WorkflowMode.KNOWLEDGE_LOOP, WorkflowMode.TOOL, - WorkflowMode.TOOL_LOOP].__contains__( - self.workflow_manage.flow.workflow_mode): - return self.execute(video=res, **self.node_params_serializer.data, **self.flow_params_serializer.data, - **{'history_chat_record': [], 'stream': True, 'chat_id': None, 'chat_record_id': None}) - else: - return self.execute(video=res, **self.node_params_serializer.data, **self.flow_params_serializer.data) - - def execute(self, model_id, system, prompt, dialogue_number, dialogue_type, history_chat_record, stream, - model_params_setting, - chat_record_id, - video, - model_id_type=None, model_id_reference=None, - model_setting=None, - **kwargs) -> NodeResult: - pass diff --git a/apps/application/flow/step_node/video_understand_step_node/impl/__init__.py b/apps/application/flow/step_node/video_understand_step_node/impl/__init__.py deleted file mode 100644 index 555faa26b66..00000000000 --- a/apps/application/flow/step_node/video_understand_step_node/impl/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# coding=utf-8 - -from .base_video_understand_node import BaseVideoUnderstandNode diff --git a/apps/application/flow/step_node/video_understand_step_node/impl/base_video_understand_node.py b/apps/application/flow/step_node/video_understand_step_node/impl/base_video_understand_node.py deleted file mode 100644 index ea497be27d0..00000000000 --- a/apps/application/flow/step_node/video_understand_step_node/impl/base_video_understand_node.py +++ /dev/null @@ -1,335 +0,0 @@ -# coding=utf-8 - -import time -from functools import reduce -from typing import List, Dict - -from django.db.models import QuerySet -from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage, AIMessage - -from application.flow.i_step_node import NodeResult, INode -from application.flow.step_node.video_understand_step_node.i_video_understand_node import IVideoUnderstandNode -from application.flow.tools import Reasoning -from knowledge.models import File -from models_provider.tools import get_model_instance_by_model_workspace_id - - -def _write_context(node_variable: Dict, workflow_variable: Dict, node: INode, workflow, answer: str, - reasoning_content: str): - chat_model = node_variable.get('chat_model') - message_tokens = node_variable['usage_metadata']['output_tokens'] if 'usage_metadata' in node_variable else 0 - answer_tokens = chat_model.get_num_tokens(answer) - node.context['message_tokens'] = message_tokens - node.context['answer_tokens'] = answer_tokens - node.context['answer'] = answer - node.context['history_message'] = node_variable['history_message'] - node.context['question'] = node_variable['question'] - node.context['run_time'] = time.time() - node.context['start_time'] - node.context['reasoning_content'] = reasoning_content - if workflow.is_result(node, NodeResult(node_variable, workflow_variable)): - node.answer_text = answer - - -def write_context_stream(node_variable: Dict, workflow_variable: Dict, node: INode, workflow): - """ - 写入上下文数据 (流式) - @param node_variable: 节点数据 - @param workflow_variable: 全局数据 - @param node: 节点 - @param workflow: 工作流管理器 - """ - response = node_variable.get('result') - answer = '' - reasoning_content = '' - model_setting = node.context.get('model_setting', - {'reasoning_content_enable': False, 'reasoning_content_end': '', - 'reasoning_content_start': ''}) - reasoning = Reasoning(model_setting.get('reasoning_content_start', ''), - model_setting.get('reasoning_content_end', '')) - response_reasoning_content = False - - for chunk in response: - if workflow.is_the_task_interrupted(): - break - - # 处理 reasoning content - reasoning_chunk = reasoning.get_reasoning_content(chunk) - content_chunk = reasoning_chunk.get('content') - if 'reasoning_content' in chunk.additional_kwargs: - response_reasoning_content = True - reasoning_content_chunk = chunk.additional_kwargs.get('reasoning_content', '') - else: - reasoning_content_chunk = reasoning_chunk.get('reasoning_content') - - answer += content_chunk - if reasoning_content_chunk is None: - reasoning_content_chunk = '' - reasoning_content += reasoning_content_chunk - - # 处理 chunk.content 为 list 的情况 - if isinstance(chunk.content, list): - for chunk_item in chunk.content: - text = chunk_item.get("text", "") - yield {'content': text, - 'reasoning_content': reasoning_content_chunk if model_setting.get('reasoning_content_enable', - False) else ''} - else: - text = chunk.content or "" - yield {'content': text, - 'reasoning_content': reasoning_content_chunk if model_setting.get('reasoning_content_enable', - False) else ''} - - reasoning_chunk = reasoning.get_end_reasoning_content() - answer += reasoning_chunk.get('content') - reasoning_content_chunk = "" - if not response_reasoning_content: - reasoning_content_chunk = reasoning_chunk.get( - 'reasoning_content') - yield {'content': reasoning_chunk.get('content'), - 'reasoning_content': reasoning_content_chunk if model_setting.get('reasoning_content_enable', - False) else ''} - _write_context(node_variable, workflow_variable, node, workflow, answer, reasoning_content) - - -def write_context(node_variable: Dict, workflow_variable: Dict, node: INode, workflow): - """ - 写入上下文数据 - @param node_variable: 节点数据 - @param workflow_variable: 全局数据 - @param node: 节点实例对象 - @param workflow: 工作流管理器 - """ - response = node_variable.get('result') - model_setting = node.context.get('model_setting', - {'reasoning_content_enable': False, 'reasoning_content_end': '', - 'reasoning_content_start': ''}) - reasoning = Reasoning(model_setting.get('reasoning_content_start'), model_setting.get('reasoning_content_end')) - reasoning_result = reasoning.get_reasoning_content(response) - reasoning_result_end = reasoning.get_end_reasoning_content() - content = reasoning_result.get('content') + reasoning_result_end.get('content') - meta = {**response.response_metadata, **response.additional_kwargs} - if 'reasoning_content' in meta: - reasoning_content = (meta.get('reasoning_content', '') or '') - else: - reasoning_content = (reasoning_result.get('reasoning_content') or '') + ( - reasoning_result_end.get('reasoning_content') or '') - _write_context(node_variable, workflow_variable, node, workflow, content, reasoning_content) - - -def file_id_to_base64(file_id: str, video_model): - file = QuerySet(File).filter(id=file_id).first() - file_bytes = file.get_bytes() - url = video_model.upload_file_and_get_url(file_bytes, file.file_name) - return url - - -class BaseVideoUnderstandNode(IVideoUnderstandNode): - def save_context(self, details, workflow_manage): - self.context['answer'] = details.get('answer') - self.context['question'] = details.get('question') - self.context['exception_message'] = details.get('err_message') - if self.node_params.get('is_result', False): - self.answer_text = details.get('answer') - - def execute(self, model_id, system, prompt, dialogue_number, dialogue_type, history_chat_record, stream, - model_params_setting, - chat_record_id, - video, - model_id_type=None, model_id_reference=None, - model_setting=None, - **kwargs) -> NodeResult: - # 处理引用类型 - if model_id_type == 'reference' and model_id_reference: - reference_data = self.workflow_manage.get_reference_field( - model_id_reference[0], - model_id_reference[1:], - ) - if reference_data and isinstance(reference_data, dict): - model_id = reference_data.get('model_id', model_id) - model_params_setting = reference_data.get('model_params_setting') - - from django.utils.translation import gettext_lazy as _ - - if model_id is None or model_id == '': - raise Exception(_('Model is not allowed to be empty')) - - workspace_id = self.workflow_manage.get_body().get('workspace_id') - if model_setting is None: - model_setting = {'reasoning_content_enable': False, 'reasoning_content_end': '', - 'reasoning_content_start': ''} - self.context['model_setting'] = model_setting - video_model = get_model_instance_by_model_workspace_id(model_id, workspace_id, - **(model_params_setting or {})) - # 执行详情中的历史消息不需要图片内容 - history_message = self.get_history_message_for_details(history_chat_record, dialogue_number) - self.context['history_message'] = history_message - system = self.workflow_manage.generate_prompt(system) - self.context['system'] = system - question = self.generate_prompt_question(prompt) - self.context['question'] = question.content - # 生成消息列表, 真实的history_message - message_list = self.generate_message_list(video_model, system, prompt, - self.get_history_message(history_chat_record, dialogue_number, - video_model), video) - self.context['message_list'] = message_list - self.generate_context_video(video) - self.context['dialogue_type'] = dialogue_type - if stream: - r = video_model.stream(message_list) - return NodeResult({'result': r, 'chat_model': video_model, 'message_list': message_list, - 'history_message': history_message, 'question': question.content}, {}, - _write_context=write_context_stream) - else: - r = video_model.invoke(message_list) - return NodeResult({'result': r, 'chat_model': video_model, 'message_list': message_list, - 'history_message': history_message, 'question': question.content}, {}, - _write_context=write_context) - - def generate_context_video(self, video): - if isinstance(video, str) and video.startswith('http'): - self.context['video_list'] = [{'url': video}] - elif video is not None and len(video) > 0: - self.context['video_list'] = video - - def get_history_message_for_details(self, history_chat_record, dialogue_number): - start_index = len(history_chat_record) - dialogue_number - history_message = reduce(lambda x, y: [*x, *y], [ - [self.generate_history_human_message_for_details(history_chat_record[index]), - self.generate_history_ai_message(history_chat_record[index])] - for index in - range(start_index if start_index > 0 else 0, len(history_chat_record))], []) - return history_message - - def generate_history_ai_message(self, chat_record): - for val in chat_record.details.values(): - if self.node.id == val['node_id'] and 'video_list' in val: - if val['dialogue_type'] == 'WORKFLOW': - return chat_record.get_ai_message() - return AIMessage(content=val.get('answer') or val.get('err_message') or '') - return chat_record.get_ai_message() - - def generate_history_human_message_for_details(self, chat_record): - for data in chat_record.details.values(): - if self.node.id == data['node_id'] and 'video_list' in data: - video_list = data['video_list'] or [] - # 增加对 None 和空列表的检查 - if not video_list or len(video_list) == 0 or data['dialogue_type'] == 'WORKFLOW': - return HumanMessage(content=chat_record.problem_text) - file_id_list = [] - url_list = [] - for image in video_list: - if 'file_id' in image: - file_id_list.append(image.get('file_id')) - elif 'url' in image: - url_list.append(image.get('url')) - return HumanMessage(content=[ - {'type': 'text', 'text': data['question']}, - *[{'type': 'video_url', 'video_url': {'url': f'./oss/file/{file_id}'}} for file_id in file_id_list], - *[{'type': 'video_url', 'video_url': {'url': url}} for url in url_list], - ]) - return HumanMessage(content=chat_record.problem_text) - - def get_history_message(self, history_chat_record, dialogue_number, video_model): - start_index = len(history_chat_record) - dialogue_number - history_message = reduce(lambda x, y: [*x, *y], [ - [self.generate_history_human_message(history_chat_record[index], video_model), - self.generate_history_ai_message(history_chat_record[index])] - for index in - range(start_index if start_index > 0 else 0, len(history_chat_record))], []) - return history_message - - def generate_history_human_message(self, chat_record, video_model): - - for data in chat_record.details.values(): - if self.node.id == data['node_id'] and 'video_list' in data: - video_list = data['video_list'] or [] - if video_list is None or len(video_list) == 0 or data['dialogue_type'] == 'WORKFLOW': - return HumanMessage(content=chat_record.problem_text) - file_id_list = [] - url_list = [] - for image in video_list: - if 'file_id' in image: - file_id_list.append(image.get('file_id')) - elif 'url' in image: - url_list.append(image.get('url')) - video_base64_list = [file_id_to_base64(video.get('file_id'), video_model) for video in video_list] - return HumanMessage( - content=[ - {'type': 'text', 'text': data['question']}, - *[{'type': 'video_url', - 'video_url': {'url': f'{base64_video}'}} for - base64_video in video_base64_list] - ]) - return HumanMessage(content=chat_record.problem_text) - - def generate_prompt_question(self, prompt): - return HumanMessage(self.workflow_manage.generate_prompt(prompt)) - - def _process_videos(self, image, video_model): - videos = [] - if isinstance(image, str) and image.startswith('http'): - videos.append({'type': 'video_url', 'video_url': {'url': image}}) - elif image is not None and len(image) > 0: - for img in image: - if 'file_id' in img: - file_id = img['file_id'] - file = QuerySet(File).filter(id=file_id).first() - url = video_model.upload_file_and_get_url(file.get_bytes(), file.file_name) - videos.append( - {'type': 'video_url', 'video_url': {'url': url}}) - elif 'url' in img and img['url'].startswith('http'): - videos.append( - {'type': 'video_url', 'video_url': {'url': img['url']}}) - return videos - - def generate_message_list(self, video_model, system: str, prompt: str, history_message, video): - prompt_text = self.workflow_manage.generate_prompt(prompt) - videos = self._process_videos(video, video_model) - - if videos: - messages = [HumanMessage(content=[{'type': 'text', 'text': prompt_text}, *videos])] - else: - messages = [HumanMessage(prompt_text)] - - if system is not None and len(system) > 0: - return [ - SystemMessage(system), - *history_message, - *messages - ] - else: - return [ - *history_message, - *messages - ] - - @staticmethod - def reset_message_list(message_list: List[BaseMessage], answer_text): - result = [{'role': 'user' if isinstance(message, HumanMessage) else 'ai', 'content': message.content} for - message - in - message_list] - result.append({'role': 'ai', 'content': answer_text}) - return result - - def get_details(self, index: int, **kwargs): - return { - 'name': self.node.properties.get('stepName'), - "index": index, - 'run_time': self.context.get('run_time'), - 'system': self.context.get('system'), - 'history_message': [{'content': message.content, 'role': message.type} for message in - (self.context.get('history_message') if self.context.get( - 'history_message') is not None else [])], - 'question': self.context.get('question'), - 'answer': self.context.get('answer'), - 'reasoning_content': self.context.get('reasoning_content'), - 'type': self.node.type, - 'message_tokens': self.context.get('message_tokens'), - 'answer_tokens': self.context.get('answer_tokens'), - 'status': self.status, - 'err_message': self.err_message, - 'video_list': self.context.get('video_list'), - 'dialogue_type': self.context.get('dialogue_type'), - 'enableException': self.node.properties.get('enableException'), - } diff --git a/apps/application/flow/tool_loop_workflow_manage.py b/apps/application/flow/tool_loop_workflow_manage.py deleted file mode 100644 index 9fc2425f014..00000000000 --- a/apps/application/flow/tool_loop_workflow_manage.py +++ /dev/null @@ -1,21 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： workflow_manage.py - @date：2024/1/9 17:40 - @desc: -""" -from application.flow.i_step_node import ToolFlowParamsSerializer -from application.flow.loop_workflow_manage import LoopWorkflowManage - - -class ToolLoopWorkflowManage(LoopWorkflowManage): - def get_params_serializer_class(self): - return ToolFlowParamsSerializer - - def get_source_type(self): - return "TOOL" - - def get_source_id(self): - return self.params.get('tool_id') diff --git a/apps/application/flow/tool_workflow_manage.py b/apps/application/flow/tool_workflow_manage.py deleted file mode 100644 index be63ca45e12..00000000000 --- a/apps/application/flow/tool_workflow_manage.py +++ /dev/null @@ -1,88 +0,0 @@ -# coding=utf-8 -""" - @project: MaxKB - @Author：虎虎 - @file： tool_workflow_manage.py - @date：2026/3/12 15:17 - @desc: -""" -import time -from concurrent.futures import ThreadPoolExecutor - -from django.db import close_old_connections -from django.utils.translation import get_language - -from application.flow.common import Workflow -from application.flow.i_step_node import WorkFlowPostHandler, ToolFlowParamsSerializer -from application.flow.workflow_manage import WorkflowManage -from common.handle.base_to_response import BaseToResponse -from common.handle.impl.response.system_to_response import SystemToResponse - -executor = ThreadPoolExecutor(max_workers=200) - - -class ToolWorkflowManage(WorkflowManage): - def __init__(self, flow: Workflow, params, work_flow_post_handler: WorkFlowPostHandler, - base_to_response: BaseToResponse = SystemToResponse(), form_data=None, - start_node_id=None, - start_node_data=None, chat_record=None, child_node=None, is_the_task_interrupted=lambda: False): - super().__init__(flow, params, work_flow_post_handler, base_to_response, form_data, None, None, None, - None, None, start_node_id, start_node_data, chat_record, child_node, is_the_task_interrupted) - self.out_context = {} - - def get_params_serializer_class(self): - return ToolFlowParamsSerializer - - def run(self): - self.context['start_time'] = time.time() - close_old_connections() - language = get_language() - if self.params.get('stream'): - return self.run_stream(self.start_node, None, language) - return self.run_block(language) - - def stream(self): - close_old_connections() - language = get_language() - self.run_chain_async(self.start_node, None, language) - return self.await_result(is_cleanup=False) - - def get_start_node(self): - return self.flow.get_node('tool-start-node') - - def get_base_node(self): - """ - 获取基础节点 - @return: - """ - return self.flow.get_node('tool-base-node') - - def get_input_field_list(self): - """ - 获取输入字段列表 - @return: 输入字段配置 - """ - base_node = self.get_base_node() - return base_node.properties.get("user_input_field_list") or [] - - def get_output_field_list(self): - """ - 获取输出字段列表配置 - @return: 输出字段列表配置 - """ - base_node = self.get_base_node() - return base_node.properties.get("user_output_field_list") or [] - - def get_input(self): - """ - 获取用户输入 - @return: 用户输入 - """ - input_field_list = self.get_input_field_list() - return {f.get('field'): self.params.get(f.get('field')) for f in input_field_list} - - def get_source_type(self): - return "TOOL" - - def get_source_id(self): - return self.params.get('tool_id') diff --git a/apps/application/flow/tools.py b/apps/application/flow/tools.py deleted file mode 100644 index 8f7dc13a9a9..00000000000 --- a/apps/application/flow/tools.py +++ /dev/null @@ -1,911 +0,0 @@ -# coding=utf-8 -""" -@project: maxkb -@Author：虎 -@file： utils.py -@date：2024/6/6 15:15 -@desc: -""" - -import asyncio -import io -import json -import os -import queue -import re -import shutil -import threading -import zipfile -from typing import Iterator - -# --------------------------------------------------------------------------- -# Fix: qwen's OpenAI-compatible streaming sends id='' (empty string) for -# intermediate tool_call_chunks while only the first chunk carries the real -# id ('call_xxx...'). langchain-core's merge_lists treats '' != 'call_xxx' as -# an ID conflict and _appends_ instead of merging → the accumulated AIMessage -# ends up with two separate tool_calls (one with empty args, one with empty -# id) instead of one correct entry. This causes the Qwen API to reject the -# next request with "function.arguments must be in JSON format". -# -# Patch: normalise id='' → None for items that have an 'index' key -# (i.e. tool_call_chunk dicts). merge_lists treats None as "no id" and will -# merge with any existing entry, keeping the real id from the first chunk. -# --------------------------------------------------------------------------- -import langchain_core.messages.ai as _lc_ai_module -import uuid_utils.compat as uuid -from asgiref.sync import sync_to_async -from common.result import result -from common.utils.logger import maxkb_logger -from common.utils.tool_code import ToolExecutor -from deepagents import create_deep_agent -from django.db.models import OuterRef, QuerySet, Subquery -from django.http import StreamingHttpResponse -from knowledge.models import File -from knowledge.models.knowledge_action import State -from langchain_core.messages import AIMessageChunk, BaseMessage, BaseMessageChunk, ToolMessage -from langchain_core.tools import StructuredTool -from langchain_core.utils._merge import merge_lists as _original_merge_lists -from langgraph.checkpoint.memory import MemorySaver -from maxkb.const import CONFIG -from pydantic import Field, create_model -from tools.models import Tool, ToolRecord, ToolType, ToolWorkflowVersion - -from application.flow.backend.sandbox_mcp import SandboxMCPBackend -from application.flow.backend.sandbox_shell import SandboxShellBackend -from application.flow.common import Workflow, WorkflowMode -from application.flow.i_step_node import ToolWorkflowPostHandler, WorkFlowPostHandler -from application.serializers.common import ToolExecute - - -def _merge_lists_normalize_empty_tool_chunk_ids(left, *others): - """Wrapper around merge_lists that normalises empty-string IDs to None in - tool_call_chunk items (those with an 'index' key) so that qwen streaming - chunks with id='' are merged correctly by index.""" - - def _norm(lst): - if lst is None: - return lst - result = [] - for item in lst: - if isinstance(item, dict) and "index" in item and item.get("id") == "": - item = {**item, "id": None} - result.append(item) - return result - - return _original_merge_lists( - _norm(left), - *[_norm(o) for o in others], - ) - - -# Replace the module-level reference used by add_ai_message_chunks in ai.py -_lc_ai_module.merge_lists = _merge_lists_normalize_empty_tool_chunk_ids - - -class Reasoning: - def __init__(self, reasoning_content_start, reasoning_content_end): - self.content = "" - self.reasoning_content = "" - self.all_content = "" - self.reasoning_content_start_tag = reasoning_content_start - self.reasoning_content_end_tag = reasoning_content_end - self.reasoning_content_start_tag_len = ( - len(reasoning_content_start) if reasoning_content_start is not None else 0 - ) - self.reasoning_content_end_tag_len = len(reasoning_content_end) if reasoning_content_end is not None else 0 - self.reasoning_content_end_tag_prefix = ( - reasoning_content_end[0] if self.reasoning_content_end_tag_len > 0 else "" - ) - self.reasoning_content_is_start = False - self.reasoning_content_is_end = False - self.reasoning_content_chunk = "" - - def get_end_reasoning_content(self): - if not self.reasoning_content_is_start and not self.reasoning_content_is_end: - r = {"content": self.all_content, "reasoning_content": ""} - self.reasoning_content_chunk = "" - return r - if self.reasoning_content_is_start and not self.reasoning_content_is_end: - r = {"content": "", "reasoning_content": self.reasoning_content_chunk} - self.reasoning_content_chunk = "" - return r - return {"content": "", "reasoning_content": ""} - - def _normalize_content(self, content): - """将不同类型的内容统一转换为字符串""" - if isinstance(content, str): - return content - elif isinstance(content, list): - # 处理包含多种内容类型的列表 - normalized_parts = [] - for item in content: - if isinstance(item, dict): - if item.get("type") == "text": - normalized_parts.append(item.get("text", "")) - return "".join(normalized_parts) - else: - return str(content) - - def get_reasoning_content(self, chunk): - # 如果没有开始思考过程标签那么就全是结果 - if self.reasoning_content_start_tag is None or len(self.reasoning_content_start_tag) == 0: - self.content += chunk.content - return {"content": chunk.content, "reasoning_content": ""} - # 如果没有结束思考过程标签那么就全部是思考过程 - if self.reasoning_content_end_tag is None or len(self.reasoning_content_end_tag) == 0: - return {"content": "", "reasoning_content": chunk.content} - chunk.content = self._normalize_content(chunk.content) - self.all_content += chunk.content - if not self.reasoning_content_is_start and len(self.all_content) >= self.reasoning_content_start_tag_len: - if self.all_content.startswith(self.reasoning_content_start_tag): - self.reasoning_content_is_start = True - self.reasoning_content_chunk = self.all_content[self.reasoning_content_start_tag_len :] - else: - if not self.reasoning_content_is_end: - self.reasoning_content_is_end = True - self.content += self.all_content - return { - "content": self.all_content, - "reasoning_content": chunk.additional_kwargs.get("reasoning_content", "") - if chunk.additional_kwargs - else "", - } - else: - if self.reasoning_content_is_start: - self.reasoning_content_chunk += chunk.content - reasoning_content_end_tag_prefix_index = self.reasoning_content_chunk.find( - self.reasoning_content_end_tag_prefix - ) - if self.reasoning_content_is_end: - self.content += chunk.content - return { - "content": chunk.content, - "reasoning_content": chunk.additional_kwargs.get("reasoning_content", "") - if chunk.additional_kwargs - else "", - } - # 是否包含结束 - if reasoning_content_end_tag_prefix_index > -1: - if ( - len(self.reasoning_content_chunk) - reasoning_content_end_tag_prefix_index - >= self.reasoning_content_end_tag_len - ): - reasoning_content_end_tag_index = self.reasoning_content_chunk.find(self.reasoning_content_end_tag) - if reasoning_content_end_tag_index > -1: - reasoning_content_chunk = self.reasoning_content_chunk[0:reasoning_content_end_tag_index] - content_chunk = self.reasoning_content_chunk[ - reasoning_content_end_tag_index + self.reasoning_content_end_tag_len : - ] - self.reasoning_content += reasoning_content_chunk - self.content += content_chunk - self.reasoning_content_chunk = "" - self.reasoning_content_is_end = True - return {"content": content_chunk, "reasoning_content": reasoning_content_chunk} - else: - reasoning_content_chunk = self.reasoning_content_chunk[ - 0 : reasoning_content_end_tag_prefix_index + 1 - ] - self.reasoning_content_chunk = self.reasoning_content_chunk.replace(reasoning_content_chunk, "") - self.reasoning_content += reasoning_content_chunk - return {"content": "", "reasoning_content": reasoning_content_chunk} - else: - return {"content": "", "reasoning_content": ""} - - else: - if self.reasoning_content_is_end: - self.content += chunk.content - return { - "content": chunk.content, - "reasoning_content": chunk.additional_kwargs.get("reasoning_content", "") - if chunk.additional_kwargs - else "", - } - else: - # aaa - result = {"content": "", "reasoning_content": self.reasoning_content_chunk} - self.reasoning_content += self.reasoning_content_chunk - self.reasoning_content_chunk = "" - return result - - -def event_content(chat_id, chat_record_id, response, workflow, write_context, post_handler: WorkFlowPostHandler): - """ - 用于处理流式输出 - @param chat_id: 会话id - @param chat_record_id: 对话记录id - @param response: 响应数据 - @param workflow: 工作流管理器 - @param write_context 写入节点上下文 - @param post_handler: 后置处理器 - """ - answer = "" - try: - for chunk in response: - answer += chunk.content - yield ( - "data: " - + json.dumps( - { - "chat_id": str(chat_id), - "id": str(chat_record_id), - "operate": True, - "content": chunk.content, - "is_end": False, - }, - ensure_ascii=False, - ) - + "\n\n" - ) - write_context(answer, 200) - post_handler.handler(chat_id, chat_record_id, answer, workflow) - yield ( - "data: " - + json.dumps( - {"chat_id": str(chat_id), "id": str(chat_record_id), "operate": True, "content": "", "is_end": True}, - ensure_ascii=False, - ) - + "\n\n" - ) - except Exception as e: - answer = str(e) - write_context(answer, 500) - post_handler.handler(chat_id, chat_record_id, answer, workflow) - yield ( - "data: " - + json.dumps( - { - "chat_id": str(chat_id), - "id": str(chat_record_id), - "operate": True, - "content": answer, - "is_end": True, - }, - ensure_ascii=False, - ) - + "\n\n" - ) - - -def to_stream_response( - chat_id, chat_record_id, response: Iterator[BaseMessageChunk], workflow, write_context, post_handler -): - """ - 将结果转换为服务流输出 - @param chat_id: 会话id - @param chat_record_id: 对话记录id - @param response: 响应数据 - @param workflow: 工作流管理器 - @param write_context 写入节点上下文 - @param post_handler: 后置处理器 - @return: 响应 - """ - r = StreamingHttpResponse( - streaming_content=event_content(chat_id, chat_record_id, response, workflow, write_context, post_handler), - content_type="text/event-stream;charset=utf-8", - charset="utf-8", - ) - - r["Cache-Control"] = "no-cache" - return r - - -def to_response( - chat_id, chat_record_id, response: BaseMessage, workflow, write_context, post_handler: WorkFlowPostHandler -): - """ - 将结果转换为服务输出 - - @param chat_id: 会话id - @param chat_record_id: 对话记录id - @param response: 响应数据 - @param workflow: 工作流管理器 - @param write_context 写入节点上下文 - @param post_handler: 后置处理器 - @return: 响应 - """ - answer = response.content - write_context(answer) - post_handler.handler(chat_id, chat_record_id, answer, workflow) - return result.success( - {"chat_id": str(chat_id), "id": str(chat_record_id), "operate": True, "content": answer, "is_end": True} - ) - - -def to_response_simple(chat_id, chat_record_id, response: BaseMessage, workflow, post_handler: WorkFlowPostHandler): - answer = response.content - post_handler.handler(chat_id, chat_record_id, answer, workflow) - return result.success( - {"chat_id": str(chat_id), "id": str(chat_record_id), "operate": True, "content": answer, "is_end": True} - ) - - -def to_stream_response_simple(stream_event): - r = StreamingHttpResponse( - streaming_content=stream_event, content_type="text/event-stream;charset=utf-8", charset="utf-8" - ) - - r["Cache-Control"] = "no-cache" - return r - - -def generate_tool_message_complete(icon, name, input_content, output_content): - """生成包含输入和输出的工具消息模版""" - # 确保输入内容是字符串，如果不是则尝试转换为 JSON 字符串 - if not isinstance(input_content, str): - input_content = json.dumps(input_content, ensure_ascii=False) - # 格式化输出 - if not isinstance(output_content, str): - output_content = json.dumps(output_content, ensure_ascii=False) - content = { - "icon": icon, - "title": name, - "type": "simple-tool-calls", - "content": {"input": input_content, "output": output_content}, - } - return f"{json.dumps(content, ensure_ascii=False)}" - - -# 全局单例事件循环 -_global_loop = None -_loop_thread = None -_loop_lock = threading.Lock() - - -def get_global_loop(): - """获取全局共享的事件循环""" - global _global_loop, _loop_thread - - with _loop_lock: - if _global_loop is None: - _global_loop = asyncio.new_event_loop() - - def run_forever(): - asyncio.set_event_loop(_global_loop) - _global_loop.run_forever() - - _loop_thread = threading.Thread(target=run_forever, daemon=True, name="GlobalAsyncLoop") - _loop_thread.start() - - return _global_loop - - -def _extract_tool_id(raw_id): - """从 raw_id 中提取最后一个符合 call_... 模式的 id，若无匹配则返回原值或 None""" - if not raw_id: - return None - if not isinstance(raw_id, str): - raw_id = str(raw_id) - - s = raw_id - prefix = "call_" - positions = [m.start() for m in re.finditer(re.escape(prefix), s)] - if not positions: - return raw_id - - # 取最后一个前缀位置，截到下一个前缀或结尾 - start = positions[-1] - end = len(s) - for pos in positions: - if pos > start: - end = pos - break - - tool_id = s[start:end] - return tool_id or raw_id - - -async def _initialize_skills(mcp_servers, temp_dir) -> SandboxMCPBackend: - skills_dir = os.path.join(temp_dir, "skills") - mcp_config = dict(mcp_servers) # Preserve server-generated InternalMCPConfig objects. - if "skills" in mcp_config: - skill_file_items = mcp_config.pop("skills") - for skill_file in skill_file_items: - # 使用 sync_to_async 包装 ORM 查询 - file = await sync_to_async(lambda: QuerySet(File).filter(id=skill_file["file_id"]).first())() - if not file: - continue - # get_bytes 可能也涉及 IO，也用 sync_to_async 包装 - file_bytes = await sync_to_async(file.get_bytes)() - params = skill_file.get("params", {}) - with zipfile.ZipFile(io.BytesIO(file_bytes), "r") as zip_ref: - members = [m for m in zip_ref.namelist() if not m.startswith("__MACOSX/") and "__MACOSX" not in m] - for member in members: - if ".." in member or member.startswith("/"): - raise ValueError(f"非法路径: {member}") - zip_ref.extractall(skills_dir, members=members) - - # 获取技能解压后的顶级目录名 - top_level_dirs = set() - for member in members: - parts = member.split("/") - if parts[0]: - top_level_dirs.add(parts[0]) - - # 将 params 写入每个顶级目录下的 .env 文件 - if params: - env_lines = [] - for key, value in params.items(): - # 对含空格或特殊字符的值加引号 - env_lines.append(f"{key}={value}") - env_content = "\n".join(env_lines) + "\n" - for top_dir in top_level_dirs: - env_path = os.path.join(skills_dir, top_dir, ".env") - with open(env_path, "w", encoding="utf-8") as f: - f.write(env_content) - - os.system("chmod -R g+rx " + temp_dir) # 确保技能目录可访问 - - return SandboxMCPBackend(mcp_config) - - -async def _yield_mcp_response( - chat_model, - system_prompt, - message_list, - mcp_servers, - mcp_output_enable=True, - tool_init_params={}, - source_id=None, - source_type=None, - temp_dir=None, - chat_id=None, - extra_tools=None, -): - try: - checkpointer = MemorySaver() - mcp_backend = await _initialize_skills(mcp_servers, temp_dir) - tools = await mcp_backend.get_tools() - for tool in tools: - tool.handle_tool_error = True - if extra_tools: - for tool in extra_tools: - tools.append(tool) - - agent = create_deep_agent( - model=chat_model, - backend=SandboxShellBackend(root_dir=temp_dir, virtual_mode=True), - skills=["/skills"], - tools=tools, - system_prompt=system_prompt, - interrupt_on={"write_file": False, "read_file": False, "edit_file": False}, - checkpointer=checkpointer, - ) - recursion_limit = int(CONFIG.get("LANGCHAIN_GRAPH_RECURSION_LIMIT", "100")) - response = agent.astream( - {"messages": message_list}, - config={"recursion_limit": recursion_limit, "configurable": {"thread_id": chat_id}}, - stream_mode="messages", - ) - - tool_calls_info = {} # tool_id -> {'name': ..., 'input': ...} - # key(index/id) -> {'id': ..., 'name': ..., 'arguments': ...} - _tool_fragments = {} - - def _merge_arguments(entry, part_args): - if not isinstance(part_args, str): - try: - part_args = json.dumps(part_args, ensure_ascii=False) - except Exception: - part_args = str(part_args) if part_args else "" - if not part_args: - return - - # Some providers first emit placeholder args like "{}" and then - # stream the real JSON fragments via later chunks. Prefer fragments. - if entry["arguments"] in ("{}", "[]") and part_args.startswith("{"): - entry["arguments"] = part_args - return - - if entry["arguments"]: - try: - existing_obj = json.loads(entry["arguments"]) - new_obj = json.loads(part_args) - if isinstance(existing_obj, dict) and isinstance(new_obj, dict): - merged = {**existing_obj, **new_obj} - entry["arguments"] = json.dumps(merged, ensure_ascii=False) - else: - entry["arguments"] += part_args - except (json.JSONDecodeError, ValueError): - entry["arguments"] += part_args - else: - entry["arguments"] = part_args - - def _get_fragment_key(idx, raw_id): - if idx is not None: - return f"idx:{idx}" - if raw_id and str(raw_id).strip(): - return f"id:{_extract_tool_id(str(raw_id).strip())}" - return None - - def _upsert_fragment(key, raw_id, func_name, part_args): - if key is None: - return - entry = _tool_fragments.setdefault(key, {"id": "", "name": "", "arguments": ""}) - - if raw_id and str(raw_id).strip(): - new_id = str(raw_id).strip() - if entry.get("completed") and entry.get("id") and entry["id"] != new_id: - maxkb_logger.debug(f"Resetting completed fragment {key}: old ID {entry['id']} -> new ID {new_id}") - entry.clear() - entry.update({"id": "", "name": "", "arguments": ""}) - entry["id"] = new_id - - if func_name: - entry["name"] = func_name - - _merge_arguments(entry, part_args) - - async for chunk in response: - # print(chunk) - if isinstance(chunk[0], AIMessageChunk): - # ---------------------------------------------------------------- - # 1. 从 tool_call_chunks 中聚合工具调用片段 - # (qwen/OpenAI streaming 通过 tool_call_chunks 传递， - # additional_kwargs['tool_calls'] 在流式时通常为空) - # ---------------------------------------------------------------- - for tc_chunk in chunk[0].tool_call_chunks or []: - raw_id = tc_chunk.get("id") - key = _get_fragment_key(tc_chunk.get("index"), raw_id) - _upsert_fragment(key, raw_id, tc_chunk.get("name"), tc_chunk.get("args", "")) - - # ---------------------------------------------------------------- - # 1.1 兼容部分模型将工具调用放在 chunk.tool_calls，且 tool_call_chunks - # 的 index 为空（例如 ollama/qwen） - # ---------------------------------------------------------------- - has_tool_call_chunks = bool(chunk[0].tool_call_chunks) - for tool_call in chunk[0].tool_calls or []: - raw_id = tool_call.get("id") - part_args = tool_call.get("args", "") - # qwen-plus often emits {} here as a placeholder while - # the real args are split in tool_call_chunks/invalid_tool_calls. - if has_tool_call_chunks and (part_args == "" or part_args == {} or part_args == []): - part_args = "" - key = _get_fragment_key(tool_call.get("index"), raw_id) - _upsert_fragment(key, raw_id, tool_call.get("name"), part_args) - - # ---------------------------------------------------------------- - # 1.2 兼容 invalid_tool_calls 分片（部分模型会把中间 JSON 片段放这里） - # ---------------------------------------------------------------- - for invalid_tool_call in chunk[0].invalid_tool_calls or []: - raw_id = invalid_tool_call.get("id") - key = _get_fragment_key(invalid_tool_call.get("index"), raw_id) - _upsert_fragment(key, raw_id, invalid_tool_call.get("name"), invalid_tool_call.get("args", "")) - - # ---------------------------------------------------------------- - # 2. 兼容 additional_kwargs['tool_calls'] 方式（旧格式/非流式情况） - # ---------------------------------------------------------------- - legacy_tool_calls = chunk[0].additional_kwargs.get("tool_calls", []) - for tool_call in legacy_tool_calls: - raw_id = tool_call.get("id") - func = tool_call.get("function", {}) - if isinstance(func, dict): - func_name = func.get("name") - part_args = func.get("arguments", "") - else: - func_name = tool_call.get("name") - part_args = tool_call.get("arguments", "") - key = _get_fragment_key(tool_call.get("index"), raw_id) - _upsert_fragment(key, raw_id, func_name, part_args) - - # ---------------------------------------------------------------- - # 3. 检测工具调用结束，更新 tool_calls_info - # ---------------------------------------------------------------- - is_finish_chunk = ( - chunk[0].response_metadata.get("finish_reason") == "tool_calls" or chunk[0].chunk_position == "last" - ) - - if is_finish_chunk: - # 在 finish chunk 时，将所有未完成的 fragment 标记完成并更新 tool_calls_info - maxkb_logger.debug(f"Processing finish chunk. Tool fragments: {_tool_fragments}") - for idx, entry in _tool_fragments.items(): - if entry.get("completed"): - maxkb_logger.debug(f"Skipping fragment {idx}: already completed") - continue - if not entry.get("id"): - maxkb_logger.debug(f"Skipping fragment {idx}: missing id. Fragment: {entry}") - continue - if not entry.get("arguments"): - maxkb_logger.debug(f"Skipping fragment {idx}: missing arguments. Fragment: {entry}") - continue - - if not entry.get("completed") and entry.get("id") and entry.get("arguments"): - try: - parsed_args = json.loads(entry["arguments"]) - filtered_args = ( - {k: v for k, v in parsed_args.items() if k not in tool_init_params} - if tool_init_params - else parsed_args - ) - normalized_id = _extract_tool_id(entry["id"]) - info = {"name": entry["name"], "input": json.dumps(filtered_args, ensure_ascii=False)} - tool_calls_info[entry["id"]] = info - if normalized_id and normalized_id != entry["id"]: - tool_calls_info[normalized_id] = info - entry["completed"] = True - maxkb_logger.debug(f"Added tool call {entry['id']} to tool_calls_info") - except (json.JSONDecodeError, ValueError) as e: - # JSON parsing failed, but still add to tool_calls_info with raw arguments - # to prevent "Tool ID not found" errors when ToolMessage arrives - maxkb_logger.warning( - f"Failed to parse tool arguments at finish for tool {entry.get('id', 'unknown')}: " - f"{entry['arguments']}, error: {e}. Using raw arguments." - ) - normalized_id = _extract_tool_id(entry["id"]) - info = { - "name": entry["name"], - # Use raw arguments - "input": entry["arguments"], - } - tool_calls_info[entry["id"]] = info - if normalized_id and normalized_id != entry["id"]: - tool_calls_info[normalized_id] = info - entry["completed"] = True - - # ---------------------------------------------------------------- - # 4. 修复 tool_call_chunks 中的空 id（回填已知 id） - # ---------------------------------------------------------------- - if chunk[0].tool_call_chunks: - for tc_chunk in chunk[0].tool_call_chunks: - key = _get_fragment_key(tc_chunk.get("index"), tc_chunk.get("id")) - if key is not None: - frag = _tool_fragments.get(key) - if frag and frag.get("id") and not tc_chunk.get("id"): - tc_chunk["id"] = frag["id"] - - # ---------------------------------------------------------------- - # 5. 修复 additional_kwargs['tool_calls']（兼容旧格式） - # 仅在 finish chunk 时写入完整参数，避免污染中间 chunk 的 - # additional_kwargs（中间 chunk 会被 ainvoke 累积，如果写入 - # 不完整 JSON 会导致下一轮 API 调用出现 arguments 非 JSON 格式错误） - # ---------------------------------------------------------------- - if legacy_tool_calls and is_finish_chunk: - fixed_tool_calls = [] - for tool_call in legacy_tool_calls: - key = _get_fragment_key(tool_call.get("index"), tool_call.get("id")) - frag = _tool_fragments.get(key) if key is not None else None - tc = dict(tool_call) - if frag and frag.get("id") and not tc.get("id"): - tc["id"] = frag["id"] - if frag and isinstance(tc.get("function"), dict): - tc["function"] = dict(tc["function"]) - if frag.get("completed"): - tc["function"]["arguments"] = frag["arguments"] - fixed_tool_calls.append(tc) - chunk[0].additional_kwargs["tool_calls"] = fixed_tool_calls - - yield chunk[0] - - if mcp_output_enable and isinstance(chunk[0], ToolMessage): - tool_id = chunk[0].tool_call_id - normalized_tool_id = _extract_tool_id(tool_id) - tool_info = tool_calls_info.get(tool_id) or tool_calls_info.get(normalized_tool_id) - - if tool_info: - try: - if isinstance(chunk[0].content, str): - tool_result = json.loads(chunk[0].content) - elif isinstance(chunk[0].content, dict): - tool_result = chunk[0].content - elif isinstance(chunk[0].content, list): - tool_result = chunk[0].content[0] if len(chunk[0].content) > 0 else {} - else: - tool_result = {} - text = tool_result.get("text") if "text" in tool_result else None - text_result = json.loads(text) if text else tool_result - if text: - tool_lib_id = text_result.pop("tool_id") if "tool_id" in text_result else None - else: - tool_lib_id = tool_result.pop("tool_id") if "tool_id" in tool_result else None - if tool_lib_id: - await save_tool_record(tool_lib_id, tool_info, tool_result, source_id, source_type) - tool_result = json.dumps(text_result, ensure_ascii=False) - except Exception as e: - tool_result = chunk[0].content - content = generate_tool_message_complete( - tool_info.get("icon", ""), tool_info["name"], tool_info["input"], tool_result - ) - chunk[0].content = content - else: - maxkb_logger.warning( - f"Tool ID {tool_id} not found in tool_calls_info. " - f"Normalized Tool ID: {normalized_tool_id}. " - f"Available IDs: {list(tool_calls_info.keys())}. " - f"Tool fragments at this point: {_tool_fragments}" - ) - - yield chunk[0] - - except ExceptionGroup as eg: - - def get_real_error(exc): - if isinstance(exc, ExceptionGroup): - return get_real_error(exc.exceptions[0]) - return exc - - real_error = get_real_error(eg) - error_msg = f"{type(real_error).__name__}: {str(real_error)}" - raise RuntimeError(error_msg) from None - - except Exception as e: - error_msg = f"{type(e).__name__}: {str(e)}" - raise RuntimeError(error_msg) from None - - -async def save_tool_record(tool_id, tool_info, tool_result, source_id, source_type): - tool = await sync_to_async(lambda: QuerySet(Tool).filter(id=tool_id).first())() - tool_info["icon"] = tool.icon - tool_record = ToolRecord( - id=uuid.uuid7(), - workspace_id=tool.workspace_id, - tool_id=tool_id, - source_type=source_type, - source_id=source_id, - meta={"input": tool_info["input"], "output": tool_result}, - state=State.SUCCESS, - ) - await sync_to_async(tool_record.save)() - - -def mcp_response_generator( - chat_model, - system_prompt, - message_list, - mcp_servers, - mcp_output_enable=True, - tool_init_params={}, - source_id=None, - source_type=None, - chat_id=None, - extra_tools=None, -): - """使用全局事件循环，不创建新实例""" - result_queue = queue.Queue() - loop = get_global_loop() # 使用共享循环 - # 创建临时文件夹 - if chat_id: - temp_dir = os.path.join("/tmp", chat_id) - else: - temp_dir = os.path.join("/tmp", str(uuid.uuid7())) - skills_dir = os.path.join(temp_dir, "skills") - os.makedirs(skills_dir, exist_ok=True) - - # print(f"Initializing skills in temporary directory: {skills_dir}") - - async def _run(): - try: - async_gen = _yield_mcp_response( - chat_model, - system_prompt, - message_list, - mcp_servers, - mcp_output_enable, - tool_init_params, - source_id, - source_type, - temp_dir, - chat_id, - extra_tools, - ) - async for chunk in async_gen: - result_queue.put(("data", chunk)) - except Exception as e: - maxkb_logger.error(f"Exception: {e}", exc_info=True) - result_queue.put(("error", e)) - finally: - result_queue.put(("done", None)) - - # 在全局循环中调度任务 - asyncio.run_coroutine_threadsafe(_run(), loop) - - while True: - msg_type, data = result_queue.get() - if msg_type == "done": - # 清理临时文件夹 - shutil.rmtree(temp_dir, ignore_errors=True) - break - if msg_type == "error": - # 清理临时文件夹 - shutil.rmtree(temp_dir, ignore_errors=True) - raise data - yield data - - -async def anext_async(agen): - return await agen.__anext__() - - -def build_schema(fields: dict): - return create_model("dynamicSchema", **fields) - - -def get_type(_type: str): - if _type == "float": - return float - if _type == "string": - return str - if _type == "int": - return int - if _type == "dict": - return dict - if _type == "array": - return list - if _type == "boolean": - return bool - return object - - -def get_workflow_args(tool, qv): - for node in qv.work_flow.get("nodes"): - if node.get("type") == "tool-base-node": - input_field_list = node.get("properties").get("user_input_field_list") - return build_schema( - { - field.get("field"): ( - get_type(field.get("type")), - Field(..., required=True, description=field.get("desc")) - if field.get("is_required") - else Field(default=None, required=False, description=field.get("desc")), - ) - for field in input_field_list - } - ) - - return build_schema({}) - - -def get_workflow_func(source_type, source_id, tool, qv, workspace_id): - tool_id = tool.id - tool_record_id = str(uuid.uuid7()) - took_execute = ToolExecute(tool_id, tool_record_id, workspace_id, source_type, source_id, False) - - def inner(**kwargs): - from application.flow.tool_workflow_manage import ToolWorkflowManage - - work_flow_manage = ToolWorkflowManage( - Workflow.new_instance(qv.work_flow, WorkflowMode.TOOL), - { - "chat_record_id": tool_record_id, - "tool_id": tool_id, - "stream": True, - "workspace_id": workspace_id, - **kwargs, - }, - ToolWorkflowPostHandler(took_execute, tool_id), - is_the_task_interrupted=lambda: False, - child_node=None, - start_node_id=None, - start_node_data=None, - chat_record=None, - ) - res = work_flow_manage.run() - for r in res: - pass - return work_flow_manage.out_context - - return inner - - -def get_tools(source_type, source_id, tool_workflow_ids, workspace_id): - tools = QuerySet(Tool).filter( - id__in=tool_workflow_ids, is_active=True, tool_type=ToolType.WORKFLOW, workspace_id=workspace_id - ) - latest_subquery = ToolWorkflowVersion.objects.filter(tool_id=OuterRef("tool_id")).order_by("-create_time") - - qs = ToolWorkflowVersion.objects.filter( - tool_id__in=[t.id for t in tools], id=Subquery(latest_subquery.values("id")[:1]) - ) - qd = {q.tool_id: q for q in qs} - results = [] - for tool in tools: - qv = qd.get(tool.id) - func = get_workflow_func(source_type, source_id, tool, qv, workspace_id) - args = get_workflow_args(tool, qv) - tool = StructuredTool.from_function( - func=func, - name=tool.name, - description=tool.desc, - args_schema=args, - ) - results.append(tool) - - return results diff --git a/apps/application/flow/workflow_manage.py b/apps/application/flow/workflow_manage.py deleted file mode 100644 index f1323c6d4b7..00000000000 --- a/apps/application/flow/workflow_manage.py +++ /dev/null @@ -1,833 +0,0 @@ -# coding=utf-8 -""" - @project: maxkb - @Author：虎 - @file： workflow_manage.py - @date：2024/1/9 17:40 - @desc: -""" -import concurrent -import json -import threading -from concurrent.futures import ThreadPoolExecutor -from functools import reduce -from typing import List, Dict - -from django.db import close_old_connections, connection -from django.utils import translation -from django.utils.translation import get_language -from langchain_core.prompts import PromptTemplate -from rest_framework import status - -from application.flow import tools -from application.flow.common import Workflow -from application.flow.i_step_node import INode, WorkFlowPostHandler, NodeResult, FlowParamsSerializer -from application.flow.step_node import get_node -from common.handle.base_to_response import BaseToResponse -from common.handle.impl.response.system_to_response import SystemToResponse -from common.utils.logger import maxkb_logger - -executor = ThreadPoolExecutor(max_workers=200) - - -class NodeResultFuture: - def __init__(self, r, e, status=200): - self.r = r - self.e = e - self.status = status - - def result(self): - if self.status == 200: - return self.r - else: - raise self.e - - -def await_result(result, timeout=1): - try: - result.result(timeout) - return False - except Exception as e: - return True - - -class NodeChunkManage: - - def __init__(self, work_flow): - self.node_chunk_list = [] - self.current_node_chunk = None - self.work_flow = work_flow - - def add_node_chunk(self, node_chunk): - self.node_chunk_list.append(node_chunk) - - def contains(self, node_chunk): - return self.node_chunk_list.__contains__(node_chunk) - - def pop(self): - if self.current_node_chunk is None: - try: - current_node_chunk = self.node_chunk_list.pop(0) - self.current_node_chunk = current_node_chunk - except IndexError as e: - pass - if self.current_node_chunk is not None: - try: - chunk = self.current_node_chunk.chunk_list.pop(0) - return chunk - except IndexError as e: - if self.current_node_chunk.is_end(): - self.current_node_chunk = None - if self.work_flow.answer_is_not_empty(): - chunk = self.work_flow.base_to_response.to_stream_chunk_response( - self.work_flow.params['chat_id'], - self.work_flow.params['chat_record_id'], - '\n\n', False, 0, 0) - self.work_flow.append_answer('\n\n') - return chunk - return self.pop() - return None - - -class WorkflowManage: - def __init__(self, flow: Workflow, params, work_flow_post_handler: WorkFlowPostHandler, - base_to_response: BaseToResponse = SystemToResponse(), form_data=None, image_list=None, - document_list=None, - audio_list=None, - video_list=None, - other_list=None, - start_node_id=None, - start_node_data=None, chat_record=None, child_node=None, is_the_task_interrupted=lambda: False): - if form_data is None: - form_data = {} - if image_list is None: - image_list = [] - if document_list is None: - document_list = [] - if audio_list is None: - audio_list = [] - if video_list is None: - video_list = [] - if other_list is None: - other_list = [] - self.start_node_id = start_node_id - self.start_node = None - self.form_data = form_data - self.image_list = image_list - self.video_list = video_list - self.document_list = document_list - self.audio_list = audio_list - self.other_list = other_list - self.params = params - self.flow = flow - self.context = {} - self.chat_context = {} - self.node_chunk_manage = NodeChunkManage(self) - self.work_flow_post_handler = work_flow_post_handler - self.current_node = None - self.current_result = None - self.answer = "" - self.answer_list = [''] - self.status = 200 - self.base_to_response = base_to_response - self.chat_record = chat_record - self.child_node = child_node - self.future_list = [] - self.lock = threading.Lock() - self.field_list = [] - self.global_field_list = [] - self.chat_field_list = [] - self.init_fields() - self.is_the_task_interrupted = is_the_task_interrupted - if start_node_id is not None: - self.load_node(chat_record, start_node_id, start_node_data) - else: - self.node_context = [] - - def init_fields(self): - field_list = [] - global_field_list = [] - chat_field_list = [] - for node in self.flow.nodes: - properties = node.properties - node_name = properties.get('stepName') - node_id = node.id - node_config = properties.get('config') - field_list.append( - {'label': '异常信息', 'value': 'exception_message', 'node_id': node_id, 'node_name': node_name}) - if node_config is not None: - fields = node_config.get('fields') - if fields is not None: - for field in fields: - field_list.append({**field, 'node_id': node_id, 'node_name': node_name}) - global_fields = node_config.get('globalFields') - if global_fields is not None: - for global_field in global_fields: - global_field_list.append({**global_field, 'node_id': node_id, 'node_name': node_name}) - chat_fields = node_config.get('chatFields') - if chat_fields is not None: - for chat_field in chat_fields: - chat_field_list.append({**chat_field, 'node_id': node_id, 'node_name': node_name}) - field_list.sort(key=lambda f: len(f.get('node_name') + f.get('value')), reverse=True) - global_field_list.sort(key=lambda f: len(f.get('node_name') + f.get('value')), reverse=True) - chat_field_list.sort(key=lambda f: len(f.get('node_name') + f.get('value')), reverse=True) - self.field_list = field_list - self.global_field_list = global_field_list - self.chat_field_list = chat_field_list - - def append_answer(self, content): - self.answer += content - self.answer_list[-1] += content - - def answer_is_not_empty(self): - return len(self.answer_list[-1]) > 0 - - def load_node(self, chat_record, start_node_id, start_node_data): - self.node_context = [] - self.answer = chat_record.answer_text - self.answer_list = chat_record.answer_text_list - self.answer_list.append('') - for node_details in sorted(chat_record.details.values(), key=lambda d: d.get('index')): - node_id = node_details.get('node_id') - if node_details.get('runtime_node_id') == start_node_id: - def get_node_params(n): - is_result = False - if ['application-node', 'loop-node', 'tool-workflow-lib-node'].__contains__(n.type): - is_result = True - return {**n.properties.get('node_data'), 'form_data': start_node_data, 'node_data': start_node_data, - 'child_node': self.child_node, 'is_result': is_result} - - self.start_node = self.get_node_cls_by_id(node_id, node_details.get('up_node_id_list'), - get_node_params=get_node_params) - self.start_node.valid_args( - {**self.start_node.node_params, 'form_data': start_node_data}, self.start_node.workflow_params) - if self.start_node.type == 'loop-node': - loop_node_data = node_details.get('loop_node_data', {}) - for k, v in node_details.get('loop_context_data').items(): - if v is not None: - self.start_node.context[k] = v - self.start_node.context['loop_node_data'] = loop_node_data - self.start_node.context['current_index'] = node_details.get('current_index') - self.start_node.context['current_item'] = node_details.get('current_item') - self.start_node.context['loop_answer_data'] = node_details.get('loop_answer_data', {}) - if self.start_node.type == 'application-node': - application_node_dict = node_details.get('application_node_dict', {}) - self.start_node.context['application_node_dict'] = application_node_dict - self.node_context.append(self.start_node) - continue - - node_id = node_details.get('node_id') - node = self.get_node_cls_by_id(node_id, node_details.get('up_node_id_list')) - node.valid_args(node.node_params, node.workflow_params) - node.save_context(node_details, self) - node.node_chunk.end() - self.node_context.append(node) - - def run(self): - close_old_connections() - language = get_language() - if self.params.get('stream'): - return self.run_stream(self.start_node, None, language) - return self.run_block(language) - - def run_block(self, language='zh'): - """ - 非流式响应 - @return: 结果 - """ - try: - self.params['stream'] = True - self.run_chain_async(None, None, language) - while self.is_run(): - pass - details = self.get_runtime_details() - message_tokens = sum([row.get('message_tokens') for row in details.values() if - 'message_tokens' in row and row.get('message_tokens') is not None]) - answer_tokens = sum([row.get('answer_tokens') for row in details.values() if - 'answer_tokens' in row and row.get('answer_tokens') is not None]) - answer_text_list = self.get_answer_text_list() - answer_text = '\n\n'.join( - '\n\n'.join([a.get('content') for a in answer]) for answer in - answer_text_list) - answer_list = reduce(lambda pre, _n: [*pre, *_n], answer_text_list, []) - self.work_flow_post_handler.handler(self) - - res = self.base_to_response.to_block_response(self.params['chat_id'], - self.params['chat_record_id'], answer_text, True - , message_tokens, answer_tokens, - _status=status.HTTP_200_OK if self.status == 200 else status.HTTP_500_INTERNAL_SERVER_ERROR, - other_params={'answer_list': answer_list}) - finally: - self._cleanup() - return res - - def _cleanup(self): - """清理所有对象引用""" - # 清理列表 - self.future_list.clear() - self.field_list.clear() - self.global_field_list.clear() - self.chat_field_list.clear() - self.image_list.clear() - self.video_list.clear() - self.document_list.clear() - self.audio_list.clear() - self.other_list.clear() - if hasattr(self, 'node_context'): - self.node_context.clear() - - # 清理字典 - self.context.clear() - self.chat_context.clear() - self.form_data.clear() - - # 清理对象引用 - self.node_chunk_manage = None - self.work_flow_post_handler = None - self.flow = None - self.start_node = None - self.current_node = None - self.current_result = None - self.chat_record = None - self.base_to_response = None - self.params = None - self.lock = None - - def run_stream(self, current_node, node_result_future, language='zh'): - """ - 流式响应 - @return: - """ - self.run_chain_async(current_node, node_result_future, language) - return tools.to_stream_response_simple(self.await_result()) - - def get_body(self): - return self.params - - def is_run(self, timeout=0.5): - future_list_len = len(self.future_list) - try: - r = concurrent.futures.wait(self.future_list, timeout) - if len(r.not_done) > 0: - return True - else: - if future_list_len == len(self.future_list): - return False - else: - return True - except Exception as e: - return True - - def await_result(self, is_cleanup=True): - try: - while self.is_run(): - while True: - chunk = self.node_chunk_manage.pop() - if chunk is not None: - yield chunk - else: - break - while True: - chunk = self.node_chunk_manage.pop() - if chunk is None: - break - yield chunk - finally: - while self.is_run(): - pass - details = self.get_runtime_details() - message_tokens = sum([row.get('message_tokens') for row in details.values() if - 'message_tokens' in row and row.get('message_tokens') is not None]) - answer_tokens = sum([row.get('answer_tokens') for row in details.values() if - 'answer_tokens' in row and row.get('answer_tokens') is not None]) - self.work_flow_post_handler.handler(self) - yield self.base_to_response.to_stream_chunk_response(self.params.get('chat_id'), - self.params.get('chat_record_id'), - '', - [], - '', True, message_tokens, answer_tokens, {}) - if is_cleanup: - self._cleanup() - - def run_chain_async(self, current_node, node_result_future, language='zh'): - future = executor.submit(self.run_chain_manage, current_node, node_result_future, language) - self.future_list.append(future) - - def run_chain_manage(self, current_node, node_result_future, language='zh'): - translation.activate(language) - if current_node is None: - start_node = self.get_start_node() - current_node = get_node(start_node.type, self.flow.workflow_mode)(start_node, self.params, self) - self.node_chunk_manage.add_node_chunk(current_node.node_chunk) - # 添加节点 - self.append_node(current_node) - result = self.run_chain(current_node, node_result_future) - if result is None: - return - node_list = self.get_next_node_list(current_node, result) - if len(node_list) == 1: - self.run_chain_manage(node_list[0], None, language) - elif len(node_list) > 1: - sorted_node_run_list = sorted(node_list, key=lambda n: n.node.y) - # 获取到可执行的子节点 - result_list = [{'node': node, 'future': executor.submit(self.run_chain_manage, node, None, language)} for - node in - sorted_node_run_list] - for r in result_list: - self.future_list.append(r.get('future')) - - def run_chain(self, current_node, node_result_future=None): - if node_result_future is None: - node_result_future = self.run_node_future(current_node) - try: - is_stream = self.params.get('stream', True) - result = self.hand_event_node_result(current_node, - node_result_future) if is_stream else self.hand_node_result( - current_node, node_result_future) - return result - except Exception as e: - maxkb_logger.error(f'Exception: {e}', exc_info=True) - return None - - def hand_node_result(self, current_node, node_result_future): - try: - current_result = node_result_future.result() - result = current_result.write_context(current_node, self) - if result is not None: - # 阻塞获取结果 - list(result) - return current_result - except Exception as e: - maxkb_logger.error(f'Exception: {e}', exc_info=True) - self.status = 500 - current_node.get_write_error_context(e) - self.answer += str(e) - finally: - current_node.node_chunk.end() - - def append_node(self, current_node): - for index in range(len(self.node_context)): - n = self.node_context[index] - if current_node.id == n.node.id and current_node.runtime_node_id == n.runtime_node_id: - self.node_context[index] = current_node - return - self.node_context.append(current_node) - - def hand_event_node_result(self, current_node, node_result_future): - runtime_node_id = current_node.runtime_node_id - real_node_id = current_node.runtime_node_id - child_node = {} - view_type = current_node.view_type - try: - self.send_progress(current_node) - current_result = node_result_future.result() - result = current_result.write_context(current_node, self) - if result is not None: - if self.is_result(current_node, current_result): - for r in result: - reasoning_content = '' - content = r - child_node = {} - node_is_end = False - view_type = current_node.view_type - node_type = current_node.type - node_name = current_node.node.properties.get('stepName') - if isinstance(r, dict): - content = r.get('content') - child_node = {'runtime_node_id': r.get('runtime_node_id'), - 'chat_record_id': r.get('chat_record_id') - , 'child_node': r.get('child_node')} - if r.__contains__('real_node_id'): - real_node_id = r.get('real_node_id') - if r.__contains__('node_is_end'): - node_is_end = r.get('node_is_end') - if r.__contains__('node_type'): - node_type = r.get("node_type") - if r.__contains__('node_name'): - node_name = r.get('node_name') - view_type = r.get('view_type') - reasoning_content = r.get('reasoning_content') - chunk = self.base_to_response.to_stream_chunk_response(self.params.get('chat_id'), - self.params.get('chat_record_id'), - current_node.id, - current_node.up_node_id_list, - content, False, 0, 0, - {'node_type': node_type, - 'runtime_node_id': runtime_node_id, - 'node_name': node_name, - 'view_type': view_type, - 'child_node': child_node, - 'node_is_end': node_is_end, - 'real_node_id': real_node_id, - 'reasoning_content': reasoning_content, - 'node_status': "SUCCESS"}) - current_node.node_chunk.add_chunk(chunk) - chunk = (self.base_to_response - .to_stream_chunk_response(self.params.get('chat_id'), - self.params.get('chat_record_id'), - current_node.id, - current_node.up_node_id_list, - '', False, 0, 0, {'node_is_end': True, - 'runtime_node_id': runtime_node_id, - 'node_type': current_node.type, - 'view_type': view_type, - 'child_node': child_node, - 'real_node_id': real_node_id, - 'reasoning_content': '', - 'node_status': "SUCCESS"})) - current_node.node_chunk.add_chunk(chunk) - else: - list(result) - if current_node.status == 500: - enableException = current_node.node.properties.get('enableException') - if not enableException: - return None - current_node.context['exception_message'] = current_node.err_message - current_node.context['branch_id'] = 'exception' - r = NodeResult({'branch_id': 'exception', 'exception': current_node.err_message}, {}, - _is_interrupt=lambda node, step_variable, global_variable: False) - r.write_context(current_node, self) - return r - if self.is_the_task_interrupted(): - current_node.status = 201 - return None - return current_result - except Exception as e: - # 添加节点 - maxkb_logger.error(f'Exception: {e}', exc_info=True) - enableException = current_node.node.properties.get('enableException') - current_node.get_write_error_context(e) - self.status = 500 - if self.is_the_task_interrupted(): - current_node.status = 201 - return None - if not enableException: - chunk = self.base_to_response.to_stream_chunk_response(self.params.get('chat_id'), - self.params.get('chat_record_id'), - current_node.id, - current_node.up_node_id_list, - 'Exception:' + str(e), False, 0, 0, - {'node_is_end': True, - 'runtime_node_id': current_node.runtime_node_id, - 'node_type': current_node.type, - 'view_type': current_node.view_type, - 'child_node': {}, - 'real_node_id': real_node_id, - 'node_status': 'ERROR'}) - current_node.node_chunk.add_chunk(chunk) - return None - else: - current_node.context['exception_message'] = current_node.err_message - current_node.context['branch_id'] = 'exception' - return NodeResult({'branch_id': 'exception', 'exception': current_node.err_message}, {}, - _is_interrupt=lambda node, step_variable, global_variable: False) - finally: - current_node.node_chunk.end() - # 归还链接到连接池 - connection.close() - - def send_progress(self, current_node): - runtime_node_id = current_node.runtime_node_id - real_node_id = current_node.runtime_node_id - child_node = {} - view_type = current_node.view_type - if 'form-node' != current_node.type: - chunk = self.base_to_response.to_stream_chunk_response(self.params.get('chat_id'), - self.params.get('chat_record_id'), - current_node.id, - current_node.up_node_id_list, - '', False, 0, 0, - {'node_type': current_node.type, - 'runtime_node_id': runtime_node_id, - 'node_name': current_node.node.properties.get( - 'stepName'), - 'view_type': view_type, - 'child_node': child_node, - 'node_is_end': True, - 'real_node_id': real_node_id, - 'reasoning_content': '', - 'node_status': "SUCCESS"}) - current_node.node_chunk.add_chunk(chunk) - - def run_node_async(self, node): - future = executor.submit(self.run_node, node) - return future - - def run_node_future(self, node): - try: - node.valid_args(node.node_params, node.workflow_params) - self.send_progress(node) - result = self.run_node(node) - return NodeResultFuture(result, None, 200) - except Exception as e: - return NodeResultFuture(None, e, 500) - - def run_node(self, node): - result = node.run() - return result - - def is_result(self, current_node, current_node_result): - return current_node.node_params.get('is_result', not self._has_next_node( - current_node, current_node_result)) if current_node.node_params is not None else False - - def get_chat_info(self): - return self.work_flow_post_handler.chat_info - - def get_chunk_content(self, chunk, is_end=False): - return 'data: ' + json.dumps( - {'chat_id': self.params['chat_id'], 'id': self.params['chat_record_id'], 'operate': True, - 'content': chunk, 'is_end': is_end}, ensure_ascii=False) + "\n\n" - - def _has_next_node(self, current_node, node_result: NodeResult | None): - """ - 是否有下一个可运行的节点 - """ - next_edge_node_list = self.flow.get_next_edge_nodes(current_node.id) or [] - for next_edge_node in next_edge_node_list: - if node_result is not None and node_result.is_assertion_result(): - edge = next_edge_node.edge - if (edge.sourceNodeId == current_node.id and - f"{edge.sourceNodeId}_{node_result.node_variable.get('branch_id')}_right" == edge.sourceAnchorId): - return True - return len(next_edge_node_list) > 0 - - def has_next_node(self, node_result: NodeResult | None): - """ - 是否有下一个可运行的节点 - """ - return self._has_next_node(self.get_start_node() if self.current_node is None else self.current_node, - node_result) - - def get_runtime_details(self, get_details=lambda n, index: n.get_details(index)): - details_result = {} - for index in range(len(self.node_context)): - node = self.node_context[index] - if self.chat_record is not None and self.chat_record.details is not None and self.start_node: - details = self.chat_record.details.get(node.runtime_node_id) - if details is not None and self.start_node.runtime_node_id != node.runtime_node_id: - details_result[node.runtime_node_id] = details - continue - details = get_details(node, index) - details['node_id'] = node.id - details['up_node_id_list'] = node.up_node_id_list - details['runtime_node_id'] = node.runtime_node_id - details_result[node.runtime_node_id] = details - return details_result - - def get_record_answer_list(self): - answer_text_list = self.get_answer_text_list() - return reduce(lambda pre, _n: [*pre, *_n], answer_text_list, []) - - def get_answer_text_list(self): - result = [] - answer_list = reduce(lambda x, y: [*x, *y], - [n.get_answer_list() for n in self.node_context if n.get_answer_list() is not None], - []) - up_node = None - for index in range(len(answer_list)): - current_answer = answer_list[index] - if len(current_answer.content) > 0: - if up_node is None or current_answer.view_type == 'single_view' or ( - current_answer.view_type == 'many_view' and up_node.view_type == 'single_view'): - result.append([current_answer]) - else: - if len(result) > 0: - exec_index = len(result) - 1 - if isinstance(result[exec_index], list): - result[exec_index].append(current_answer) - else: - result.insert(0, [current_answer]) - up_node = current_answer - if len(result) == 0: - # 如果没有响应 就响应一个空数据 - return [[]] - return [[item.to_dict() for item in r] for r in result] - - @staticmethod - def dependent_node(edge, node): - up_node_id = edge.sourceNodeId - if not node.node_chunk.is_end(): - return False - if node.id == up_node_id: - if node.context.get('branch_id', None): - if edge.sourceAnchorId == f"{node.id}_{node.context.get('branch_id', None)}_right": - return True - else: - return False - if node.type == 'form-node': - if node.context.get('form_data', None) is not None: - return True - return False - return True - - def dependent_node_been_executed(self, node_id): - """ - 判断依赖节点是否都已执行 - @param node_id: 需要判断的节点id - @return: - """ - up_edge_list = [edge for edge in self.flow.edges if edge.targetNodeId == node_id] - return all( - [any([self.dependent_node(edge, node) for node in self.node_context if node.id == edge.sourceNodeId]) for - edge in - up_edge_list]) - - def get_next_node_list(self, current_node, current_node_result): - """ - 获取下一个可执行节点列表 - @param current_node: 当前可执行节点 - @param current_node_result: 当前可执行节点结果 - @return: 可执行节点列表 - """ - # 判断是否中断执行 - if current_node_result.is_interrupt_exec(current_node): - return [] - node_list = [] - next_edge_node_list = self.flow.get_next_edge_nodes(current_node.id) or [] - if current_node_result is not None and current_node_result.is_assertion_result(): - for edge_node in next_edge_node_list: - edge = edge_node.edge - next_node = edge_node.node - if ( - f"{edge.sourceNodeId}_{current_node_result.node_variable.get('branch_id')}_right" == edge.sourceAnchorId): - if next_node.properties.get('condition', "AND") == 'AND': - if self.dependent_node_been_executed(edge.targetNodeId): - up_nodes = self.flow.get_up_nodes(edge.targetNodeId) - up_node_id_list = [*current_node.up_node_id_list, current_node.node.id] - if up_nodes and len(up_nodes) > 1: - up_nodes.sort(key=lambda node: node.id) - first = up_nodes[0] - up_node_id_list = [n_c for n_c in self.node_context if n_c.node.id == first.id][ - 0].up_node_id_list - up_node_id_list = [*up_node_id_list, first.id] - node_list.append( - self.get_node_cls_by_id(edge.targetNodeId, - up_node_id_list)) - else: - node_list.append( - self.get_node_cls_by_id(edge.targetNodeId, - [*current_node.up_node_id_list, current_node.node.id])) - else: - for edge_node in next_edge_node_list: - edge = edge_node.edge - if edge.sourceNodeId + '_right' == edge.sourceAnchorId: - next_node = edge_node.node - if next_node.properties.get('condition', "AND") == 'AND': - if self.dependent_node_been_executed(edge.targetNodeId): - up_nodes = self.flow.get_up_nodes(edge.targetNodeId) - up_node_id_list = [*current_node.up_node_id_list, current_node.node.id] - if up_nodes and len(up_nodes) > 1: - up_nodes.sort(key=lambda node: node.id) - first = up_nodes[0] - up_node_id_list = [n_c for n_c in self.node_context if n_c.node.id == first.id][ - 0].up_node_id_list - up_node_id_list = [*up_node_id_list, first.id] - node_list.append( - self.get_node_cls_by_id(edge.targetNodeId, - up_node_id_list)) - else: - node_list.append( - self.get_node_cls_by_id(edge.targetNodeId, - [*current_node.up_node_id_list, current_node.node.id])) - return [node for node in node_list if not node.node.properties.get('disabled')] - - def get_reference_field(self, node_id: str, fields: List[str]): - """ - @param node_id: 节点id - @param fields: 字段 - @return: - """ - if node_id == 'global': - return INode.get_field(self.context, fields) - elif node_id == 'chat': - return INode.get_field(self.chat_context, fields) - else: - node = self.get_node_by_id(node_id) - if node: - return node.get_reference_field(fields) - return None - - def get_workflow_content(self): - context = { - 'global': self.context, - 'chat': self.chat_context - } - - for node in self.node_context: - context[node.id] = node.context - return context - - def reset_prompt(self, prompt: str): - placeholder = "{}" - for field in self.field_list: - globeLabel = f"{field.get('node_name')}.{field.get('value')}" - globeValue = f"context.get('{field.get('node_id')}',{placeholder}).get('{field.get('value', '')}','')" - prompt = prompt.replace(globeLabel, globeValue) - for field in self.global_field_list: - globeLabel = f"全局变量.{field.get('value')}" - globeLabelNew = f"global.{field.get('value')}" - globeValue = f"context.get('global').get('{field.get('value', '')}','')" - prompt = prompt.replace(globeLabel, globeValue).replace(globeLabelNew, globeValue) - for field in self.chat_field_list: - chatLabel = f"chat.{field.get('value')}" - chatValue = f"context.get('chat').get('{field.get('value', '')}','')" - prompt = prompt.replace(chatLabel, chatValue) - - return prompt - - def generate_prompt(self, prompt: str): - """ - 格式化生成提示词 - @param prompt: 提示词信息 - @return: 格式化后的提示词 - """ - context = self.get_workflow_content() - prompt = self.reset_prompt(prompt) - prompt_template = PromptTemplate.from_template(prompt, template_format='jinja2') - value = prompt_template.format(context=context) - return value - - def get_start_node(self): - """ - 获取启动节点 - @return: - """ - start_node_list = [node for node in self.flow.nodes if node.type == 'start-node'] - return start_node_list[0] - - def get_base_node(self): - """ - 获取基础节点 - @return: - """ - base_node_list = [node for node in self.flow.nodes if node.type == 'base-node'] - return base_node_list[0] - - def get_node_cls_by_id(self, node_id, up_node_id_list=None, - get_node_params=lambda node: node.properties.get('node_data')): - for node in self.flow.nodes: - if node.id == node_id: - node_instance = get_node(node.type, self.flow.workflow_mode)(node, - self.params, self, up_node_id_list, - get_node_params) - return node_instance - return None - - def get_node_by_id(self, node_id): - for node in self.node_context: - if node.id == node_id: - return node - return None - - def get_node_reference(self, reference_address: Dict): - node = self.get_node_by_id(reference_address.get('node_id')) - return node.context[reference_address.get('node_field')] - - def get_params_serializer_class(self): - return FlowParamsSerializer - - def get_source_type(self): - return "APPLICATION" - - def get_source_id(self): - return self.params.get('application_id') diff --git a/apps/application/serializers/application.py b/apps/application/serializers/application.py index 50f5e9aad58..a115a98bcc5 100644 --- a/apps/application/serializers/application.py +++ b/apps/application/serializers/application.py @@ -21,7 +21,7 @@ import requests import uuid_utils.compat as uuid -from application.flow.common import Workflow +from application.workflow.common import Workflow from application.long_term_memory import schedule_extract_long_term_memory from application.models.application import Application, ApplicationFolder, ApplicationTypeChoices, ApplicationVersion from application.models.application_access_token import ApplicationAccessToken @@ -51,7 +51,7 @@ from knowledge.models import File, FileSourceType, Knowledge, KnowledgeScope from knowledge.serializers.common import BatchMoveSerializer, BatchSerializer from knowledge.serializers.knowledge import KnowledgeModelSerializer, KnowledgeSerializer -from application.flow.backend.sandbox_mcp import SandboxMCPBackend +from application.workflow.backend.sandbox_mcp import SandboxMCPBackend from maxkb.conf import PROJECT_DIR from maxkb.const import CONFIG from models_provider.models import Model diff --git a/apps/application/flow/backend/sandbox_mcp.py b/apps/application/workflow/backend/sandbox_mcp.py similarity index 100% rename from apps/application/flow/backend/sandbox_mcp.py rename to apps/application/workflow/backend/sandbox_mcp.py diff --git a/apps/chat/serializers/chat.py b/apps/chat/serializers/chat.py index 75e75d1d21a..da2d71ff17b 100644 --- a/apps/chat/serializers/chat.py +++ b/apps/chat/serializers/chat.py @@ -23,7 +23,7 @@ from rest_framework import serializers from rest_framework.request import Request -from application.flow.tools import to_stream_response_simple +from common.utils.common import to_stream_response_simple from application.models import ( Application, ApplicationVersion, diff --git a/apps/common/mcp/client.py b/apps/common/mcp/client.py index 0b83bb77023..767af83f15b 100644 --- a/apps/common/mcp/client.py +++ b/apps/common/mcp/client.py @@ -1,6 +1,6 @@ """Compatibility exports and client factory for the dedicated MCP backend.""" -from application.flow.backend.sandbox_mcp import SandboxMCPBackend +from application.workflow.backend.sandbox_mcp import SandboxMCPBackend from common.mcp.config import InternalMCPConfig, validate_mcp_servers diff --git a/apps/common/utils/common.py b/apps/common/utils/common.py index 56672a919b2..df589b220bb 100644 --- a/apps/common/utils/common.py +++ b/apps/common/utils/common.py @@ -24,6 +24,7 @@ from django.contrib.auth.hashers import check_password, make_password from django.core.files.uploadedfile import InMemoryUploadedFile from django.db.models import QuerySet +from django.http import StreamingHttpResponse from django.utils.translation import gettext as _ from maxkb.settings import TIME_ZONE from openpyxl.cell.cell import ILLEGAL_CHARACTERS_RE @@ -524,3 +525,12 @@ def reset_value(value): c = datetime.timezone(eastern._utcoffset) value = value.astimezone(c) return value + + +def to_stream_response_simple(stream_event): + r = StreamingHttpResponse( + streaming_content=stream_event, content_type="text/event-stream;charset=utf-8", charset="utf-8" + ) + + r["Cache-Control"] = "no-cache" + return r diff --git a/apps/knowledge/tests.py b/apps/knowledge/tests.py index 785f5674a57..4c9b8af47c3 100644 --- a/apps/knowledge/tests.py +++ b/apps/knowledge/tests.py @@ -2,7 +2,6 @@ from io import BytesIO from unittest.mock import MagicMock, patch -from application.flow.i_step_node import KnowledgeWorkflowPostHandler from common.exception.app_exception import AppApiException from django.core.files.uploadedfile import SimpleUploadedFile from django.test import SimpleTestCase @@ -48,7 +47,7 @@ KnowledgeSyncSettingOperationSerializer, KnowledgeSyncSettingRequest, ) -from knowledge.serializers.knowledge_workflow import KnowledgeWorkflowActionSerializer +from knowledge.serializers.knowledge_workflow import KnowledgeWorkflowActionSerializer, finalize_knowledge_action from knowledge.serializers.problem import ProblemInstanceSerializer, ProblemSerializer from knowledge.services.document_strategy import ( apply_length_strategy, @@ -1047,12 +1046,9 @@ def test_generic_scheduled_entry_dispatches_workflow_task(self, query_set, delay class WorkflowKnowledgeScheduleTests(SimpleTestCase): - @patch("application.flow.i_step_node.merge_workflow_incremental_snapshot") - @patch("application.flow.i_step_node.get_workflow_state", return_value=KnowledgeActionState.SUCCESS) - @patch("application.flow.i_step_node.QuerySet") - def test_incremental_workflow_uses_stable_snapshot_merge( - self, query_set, _get_workflow_state, merge_workflow_snapshot - ): + @patch("knowledge.serializers.knowledge_workflow.merge_workflow_incremental_snapshot") + @patch("knowledge.serializers.knowledge_workflow.QuerySet") + def test_incremental_workflow_uses_stable_snapshot_merge(self, query_set, merge_workflow_snapshot): sync_log = MagicMock( id="00000000-0000-0000-0000-000000000032", knowledge_id="00000000-0000-0000-0000-000000000033", @@ -1070,15 +1066,16 @@ def test_incremental_workflow_uses_stable_snapshot_merge( "deleted_count": 0, "failed_count": 0, } - workflow = MagicMock(context={"start_time": timezone.now().timestamp()}) document_cleanup = MagicMock() - KnowledgeWorkflowPostHandler( - None, + # 新引擎:完成收尾由 finalize_knowledge_action 内联处理,state/run_time 由调用方算好传入 + finalize_knowledge_action( "00000000-0000-0000-0000-000000000036", + KnowledgeActionState.SUCCESS, + 0.0, str(sync_log.id), document_cleanup, - ).handler(workflow) + ) merge_workflow_snapshot.assert_called_once_with(sync_log) update = log_query.filter.return_value.update.call_args.kwargs diff --git a/apps/tools/serializers/tool.py b/apps/tools/serializers/tool.py index b0f2886300c..a9ca59b8956 100644 --- a/apps/tools/serializers/tool.py +++ b/apps/tools/serializers/tool.py @@ -36,7 +36,7 @@ from django.utils.translation import gettext_lazy as _ from knowledge.models import File, FileSourceType, Knowledge from langchain_core.messages import AIMessage, HumanMessage -from application.flow.backend.sandbox_mcp import SandboxMCPBackend +from application.workflow.backend.sandbox_mcp import SandboxMCPBackend from maxkb.const import CONFIG, PROJECT_DIR from models_provider.models import Model from rest_framework import serializers, status @@ -1593,7 +1593,7 @@ class GenerateCodeSerializer(serializers.Serializer): input_field_list = serializers.ListField(required=False, default=list, label=_("Input Field List")) def generate_code(self): - from application.flow.tools import to_stream_response_simple + from common.utils.common import to_stream_response_simple from models_provider.tools import get_model_instance_by_model_workspace_id self.is_valid(raise_exception=True) diff --git a/apps/tools/serializers/tool_workflow.py b/apps/tools/serializers/tool_workflow.py index b6794ac9bf5..3212b3315a8 100644 --- a/apps/tools/serializers/tool_workflow.py +++ b/apps/tools/serializers/tool_workflow.py @@ -22,7 +22,7 @@ import requests import uuid_utils.compat as uuid -from application.flow.tools import to_stream_response_simple +from common.utils.common import to_stream_response_simple from application.workflow.common import WorkflowType, new_instance from application.workflow.message.aggregator import AggregationManager from application.workflow.nodes import get_node_class diff --git a/apps/trigger/handler/impl/task/tool_task/workflow_tool_task.py b/apps/trigger/handler/impl/task/tool_task/workflow_tool_task.py index 9ef050fb92d..26968f687fe 100644 --- a/apps/trigger/handler/impl/task/tool_task/workflow_tool_task.py +++ b/apps/trigger/handler/impl/task/tool_task/workflow_tool_task.py @@ -1,20 +1,23 @@ # coding=utf-8 """ - @project: MaxKB - @Author：虎虎 - @file： workflow_tool_task.py.py - @date：2026/3/27 18:47 - @desc: +@project: MaxKB +@Author：虎虎 +@file： workflow_tool_task.py.py +@date：2026/3/27 18:47 +@desc: """ + +import threading import time import traceback import uuid_utils.compat as uuid from django.db.models import QuerySet -from application.flow.common import WorkflowMode, Workflow -from application.flow.i_step_node import ToolWorkflowPostHandler, get_tool_workflow_state -from application.serializers.common import ToolExecute +from application.workflow.common import WorkflowType, get_node_parameters, new_instance +from application.workflow.nodes import get_node_class +from application.workflow.status import Status +from application.workflow.workflow_manage import CallBack, WorkflowManage from common.utils.common import common_convert_value from common.utils.logger import maxkb_logger from common.utils.tool_code import ToolExecutor @@ -37,11 +40,12 @@ def get_reference(fields, obj): def get_field_value(value, kwargs): - source = value.get('source') - if source == 'custom': - return value.get('value') + source = value.get("source") + if source == "custom": + return value.get("value") else: - return get_reference(value.get('value'), kwargs) + return get_reference(value.get("value"), kwargs) + def get_tool_execute_parameters(input_field_list, parameter_setting, kwargs): type_map = {f.get("name"): f.get("type") for f in (input_field_list or []) if f.get("name")} @@ -59,79 +63,97 @@ def support(self, tool, trigger_task, **kwargs): return tool.tool_type == ToolType.WORKFLOW def execute(self, tool, trigger_task, **kwargs): - parameter_setting = trigger_task.get('parameter') - tool_id = trigger_task.get('source_id') + parameter_setting = trigger_task.get("parameter") + tool_id = trigger_task.get("source_id") task_record_id = uuid.uuid7() start_time = time.time() try: TaskRecord( id=task_record_id, - trigger_id=trigger_task.get('trigger'), - trigger_task_id=trigger_task.get('id'), + trigger_id=trigger_task.get("trigger"), + trigger_task_id=trigger_task.get("id"), source_type="TOOL", source_id=tool_id, task_record_id=task_record_id, - meta={'input': parameter_setting, 'output': {}}, - state=State.STARTED + meta={"input": parameter_setting, "output": {}}, + state=State.STARTED, ).save() ToolRecord( id=task_record_id, workspace_id=tool.workspace_id, tool_id=tool.id, source_type=ToolTaskTypeChoices.TRIGGER, - source_id=trigger_task.get('trigger'), - meta={'input': parameter_setting, 'output': {}}, - state=State.STARTED + source_id=trigger_task.get("trigger"), + meta={"input": parameter_setting, "output": {}}, + state=State.STARTED, ).save() - tool_workflow_version = QuerySet(ToolWorkflowVersion).filter(tool_id=tool.id).order_by( - '-create_time')[0:1].first() + tool_workflow_version = ( + QuerySet(ToolWorkflowVersion).filter(tool_id=tool.id).order_by("-create_time")[0:1].first() + ) if not tool_workflow_version: maxkb_logger.info(f"Tool with id {tool_id} not found or inactive.") return - flow = Workflow.new_instance(tool_workflow_version.work_flow, WorkflowMode.TOOL) - base_node = flow.get_node('tool-base-node') + workflow = new_instance(tool_workflow_version.work_flow, WorkflowType.TOOL) + base_node = workflow.get_node("tool-base-node") user_input_field_list = base_node.properties.get("user_input_field_list") or [] - parameters = get_tool_execute_parameters(user_input_field_list, - parameter_setting.get('user_input_field_list'), kwargs) - took_execute = ToolExecute(tool_id, str(task_record_id), - tool.workspace_id, - ToolTaskTypeChoices.TRIGGER, - trigger_task.get('trigger'), - False) - from application.flow.tool_workflow_manage import ToolWorkflowManage - work_flow_manage = ToolWorkflowManage( - flow, - { - 'chat_record_id': task_record_id, - 'tool_id': tool_id, - 'stream': True, - 'workspace_id': tool.workspace_id, - **parameters}, - ToolWorkflowPostHandler(took_execute, tool_id), - is_the_task_interrupted=lambda: False, - child_node=None, - start_node_id=None, - start_node_data=None, - chat_record=None + field_parameters = get_tool_execute_parameters( + user_input_field_list, parameter_setting.get("user_input_field_list"), kwargs ) - res = work_flow_manage.run() - for r in res: + # 对齐旧引擎 body:输入字段值 + 运行身份;新引擎 tool-start-node 按 field 从这里取值 + parameters = { + "tool_id": tool_id, + "stream": True, + "workspace_id": tool.workspace_id, + **field_parameters, + } + + # 后台任务:非流式,run() 起线程异步执行,完成后经 on_complete 通知,这里阻塞等结果 + done_event = threading.Event() + run_result = {"error": None} + + def on_next(wf_manage, content): pass - state = get_tool_workflow_state(work_flow_manage) + + def on_complete(wf_manage, error): + run_result["error"] = error + done_event.set() + + call_back = CallBack(on_next, on_complete) + + def get_start_node_fn(wf, wm): + start_node = wf.get_node("tool-start-node") + if start_node is None: + raise Exception("The start node does not exist") + node_class = get_node_class(start_node.type, WorkflowType.TOOL) + return node_class(start_node, wm, get_node_parameters) + + work_flow_manage = WorkflowManage(workflow, parameters, WorkflowType.TOOL, call_back, get_start_node_fn) + work_flow_manage.run() + done_event.wait() + + if run_result["error"]: + raise run_result["error"] + + # 新引擎工具输出收口于全局 output 上下文(tool-start-node 初始化、变量赋值节点写入) + output = work_flow_manage.context.get("output", {}) + details = work_flow_manage.get_details() + has_fail = any((d or {}).get("status") == Status.FAIL.value for d in (details or [])) + state = State.FAILURE if has_fail else State.SUCCESS QuerySet(TaskRecord).filter(id=task_record_id).update( - state=state, - run_time=time.time() - start_time, - meta={'input': parameter_setting, 'output': work_flow_manage.out_context} + state=state, run_time=time.time() - start_time, meta={"input": parameter_setting, "output": output} + ) + QuerySet(ToolRecord).filter(id=task_record_id).update( + state=state, run_time=time.time() - start_time, meta={"input": parameter_setting, "output": output} ) except Exception as e: maxkb_logger.error(f"Tool execution error: {traceback.format_exc()}") QuerySet(TaskRecord).filter(id=task_record_id).update( state=State.FAILURE, run_time=time.time() - start_time, - meta={'input': parameter_setting, 'output': 'Error: ' + str(e), 'err_message': 'Error: ' + str(e)} + meta={"input": parameter_setting, "output": "Error: " + str(e), "err_message": "Error: " + str(e)}, ) QuerySet(ToolRecord).filter(id=task_record_id).update( state=State.FAILURE, run_time=time.time() - start_time, - meta={'input': parameter_setting, 'output': 'Error: ' + str(e), 'err_message': 'Error: ' + str(e)} + meta={"input": parameter_setting, "output": "Error: " + str(e), "err_message": "Error: " + str(e)}, )